diff --git a/.gitmodules b/.gitmodules index 6cce669f7..0c2321bcf 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,4 +1,4 @@ [submodule "app/console"] path = app/console url = https://github.com/appwrite/console - branch = 3.3.15 \ No newline at end of file + branch = 3.2.16 diff --git a/Dockerfile b/Dockerfile index 2ec9fd409..324a3a548 100755 --- a/Dockerfile +++ b/Dockerfile @@ -93,8 +93,10 @@ RUN chmod +x /usr/local/bin/doctor && \ chmod +x /usr/local/bin/worker-mails && \ chmod +x /usr/local/bin/worker-messaging && \ chmod +x /usr/local/bin/worker-webhooks && \ - chmod +x /usr/local/bin/worker-usage && \ - chmod +x /usr/local/bin/worker-migrations + chmod +x /usr/local/bin/worker-migrations && \ + chmod +x /usr/local/bin/worker-hamster && \ + chmod +x /usr/local/bin/worker-usage + # Cloud Executabless RUN chmod +x /usr/local/bin/hamster && \ diff --git a/app/cli.php b/app/cli.php index f2b9b454d..d7cc5fe41 100644 --- a/app/cli.php +++ b/app/cli.php @@ -6,6 +6,7 @@ require_once __DIR__ . '/controllers/general.php'; use Appwrite\Event\Delete; use Appwrite\Event\Certificate; use Appwrite\Event\Func; +use Appwrite\Event\Hamster; use Appwrite\Platform\Appwrite; use Utopia\CLI\CLI; use Utopia\Database\Validator\Authorization; @@ -130,6 +131,9 @@ CLI::setResource('queue', function (Group $pools) { CLI::setResource('queueForFunctions', function (Connection $queue) { return new Func($queue); }, ['queue']); +CLI::setResource('queueForHamster', function (Connection $queue) { + return new Hamster($queue); +}, ['queue']); CLI::setResource('queueForDeletes', function (Connection $queue) { return new Delete($queue); }, ['queue']); diff --git a/app/config/locale/translations/tr.json b/app/config/locale/translations/tr.json index 6a94aeaca..e82317de0 100644 --- a/app/config/locale/translations/tr.json +++ b/app/config/locale/translations/tr.json @@ -3,30 +3,36 @@ "settings.locale": "tr", "settings.direction": "ltr", "emails.sender": "%s Takımı", - "emails.verification.subject": "", - "emails.verification.hello": "", - "emails.verification.body": "", - "emails.verification.footer": "", - "emails.verification.thanks": "", - "emails.verification.signature": "", - "emails.magicSession.subject": "", - "emails.magicSession.hello": "", - "emails.magicSession.body": "", - "emails.magicSession.footer": "", - "emails.magicSession.thanks": "", - "emails.magicSession.signature": "", - "emails.recovery.subject": "", - "emails.recovery.hello": "", - "emails.recovery.body": "", - "emails.recovery.footer": "", - "emails.recovery.thanks": "", - "emails.recovery.signature": "", - "emails.invitation.subject": "", - "emails.invitation.hello": "", - "emails.invitation.body": "", - "emails.invitation.footer": "", - "emails.invitation.thanks": "", - "emails.invitation.signature": "", + "emails.verification.subject": "Hesabını Doğrula", + "emails.verification.hello": "Merhaba {{user}}", + "emails.verification.body": "Eposta adresini doğrulamak için bu bağlantıyı kullanın.", + "emails.verification.footer": "Eğer bu eposta adresini doğrulamak isteyen siz değilseniz devam etmeyin.", + "emails.verification.thanks": "Teşekkürler", + "emails.verification.signature": "{{project}} takımı", + "emails.magicSession.subject": "Giriş", + "emails.magicSession.hello": "Merhaba,", + "emails.magicSession.body": "Giriş yapmak için tıklayın.", + "emails.magicSession.footer": "Eğer bu eposta adresini kullanarak giriş yapmak istemediyseniz devam etmeyin.", + "emails.magicSession.thanks": "Teşekkürler", + "emails.magicSession.signature": "{{project}} takımı", + "emails.recovery.subject": "Şifremi Sıfırla", + "emails.recovery.hello": "Merhaba {{user}}", + "emails.recovery.body": "{{project}} şifrenizi sıfırlamak için bu bağlantıyı kullanın.", + "emails.recovery.footer": "Eğer şifre sıfırlama talebinde bulunmadıysanız devam etmeyin.", + "emails.recovery.thanks": "Teşekkürler", + "emails.recovery.signature": "{{project}} takımı", + "emails.invitation.subject": "%s üzerinde %s Takımına Davet", + "emails.invitation.hello": "Merhaba", + "emails.invitation.body": "Bu epostayı aldınız, çünkü {{owner}} sizi {{project}} üzerinde {{team}} takımının üyesi olmaya davet etti.", + "emails.invitation.footer": "Eğer ilgilenmiyorsanız devam etmeyin.", + "emails.invitation.thanks": "Teşekkürler", + "emails.invitation.signature": "{{project}} takımı", + "emails.certificate.subject": "%s için sertifika hatası", + "emails.certificate.hello": "Merhaba", + "emails.certificate.body": "Alan adınız '{{domain}}' için sertifika oluşturulamadı. Deneme sayısı {{attempt}} ve hata sebebi: {{error}}", + "emails.certificate.footer": "Geçmiş sertifikanız ilk denemeden sonra 30 gün daha geçerli kalacaktır. Bu konuyu araştırmanızı öneriyoruz, aksi taktirde alan adınız SSL sertifikasız kalacaktır.", + "emails.certificate.thanks": "Teşekkürler", + "emails.certificate.signature": "{{project}} takımı", "locale.country.unknown": "Bilinmeyen", "countries.af": "Afganistan", "countries.ao": "Angola", @@ -229,4 +235,4 @@ "continents.na": "Kuzey Amerika", "continents.oc": "Okyanusya", "continents.sa": "Güney Amerika" -} \ No newline at end of file +} diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index df7d40134..61e129e93 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -549,11 +549,19 @@ App::get('/v1/account/sessions/oauth2/:provider/redirect') if (!$user->isEmpty()) { $userId = $user->getId(); - $identitiesWithMatchingEmail = $dbForProject->find('identities', [ + $identityWithMatchingEmail = $dbForProject->findOne('identities', [ Query::equal('providerEmail', [$email]), Query::notEqual('userId', $userId), ]); - if (!empty($identitiesWithMatchingEmail)) { + if (!empty($identityWithMatchingEmail)) { + throw new Exception(Exception::USER_ALREADY_EXISTS); + } + + $userWithMatchingEmail = $dbForProject->find('users', [ + Query::equal('email', [$email]), + Query::notEqual('$id', $userId), + ]); + if (!empty($userWithMatchingEmail)) { throw new Exception(Exception::USER_ALREADY_EXISTS); } } diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index 157467867..efe251546 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -44,6 +44,7 @@ use Utopia\Validator\Text; use Utopia\Validator\WhiteList; use Utopia\DSN\DSN; use Utopia\Swoole\Request; +use Utopia\Storage\Compression\Compression; App::post('/v1/storage/buckets') ->desc('Create bucket') @@ -66,7 +67,7 @@ App::post('/v1/storage/buckets') ->param('enabled', true, new Boolean(true), 'Is bucket enabled? When set to \'disabled\', users cannot access the files in this bucket but Server SDKs with and API key can still access the bucket. No files are lost when this is toggled.', true) ->param('maximumFileSize', (int) App::getEnv('_APP_STORAGE_LIMIT', 0), new Range(1, (int) App::getEnv('_APP_STORAGE_LIMIT', 0)), 'Maximum file size allowed in bytes. Maximum allowed value is ' . Storage::human(App::getEnv('_APP_STORAGE_LIMIT', 0), 0) . '.', true) ->param('allowedFileExtensions', [], new ArrayList(new Text(64), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Allowed file extensions. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' extensions are allowed, each 64 characters long.', true) - ->param('compression', COMPRESSION_TYPE_NONE, new WhiteList([COMPRESSION_TYPE_NONE, COMPRESSION_TYPE_GZIP, COMPRESSION_TYPE_ZSTD]), 'Compression algorithm choosen for compression. Can be one of ' . COMPRESSION_TYPE_NONE . ', [' . COMPRESSION_TYPE_GZIP . '](https://en.wikipedia.org/wiki/Gzip), or [' . COMPRESSION_TYPE_ZSTD . '](https://en.wikipedia.org/wiki/Zstd), For file size above ' . Storage::human(APP_STORAGE_READ_BUFFER, 0) . ' compression is skipped even if it\'s enabled', true) + ->param('compression', Compression::NONE, new WhiteList([Compression::NONE, Compression::GZIP, Compression::ZSTD]), 'Compression algorithm choosen for compression. Can be one of ' . Compression::NONE . ', [' . Compression::GZIP . '](https://en.wikipedia.org/wiki/Gzip), or [' . Compression::ZSTD . '](https://en.wikipedia.org/wiki/Zstd), For file size above ' . Storage::human(APP_STORAGE_READ_BUFFER, 0) . ' compression is skipped even if it\'s enabled', true) ->param('encryption', true, new Boolean(true), 'Is encryption enabled? For file size above ' . Storage::human(APP_STORAGE_READ_BUFFER, 0) . ' encryption is skipped even if it\'s enabled', true) ->param('antivirus', true, new Boolean(true), 'Is virus scanning enabled? For file size above ' . Storage::human(APP_LIMIT_ANTIVIRUS, 0) . ' AntiVirus scanning is skipped even if it\'s enabled', true) ->inject('response') @@ -237,7 +238,7 @@ App::put('/v1/storage/buckets/:bucketId') ->param('enabled', true, new Boolean(true), 'Is bucket enabled? When set to \'disabled\', users cannot access the files in this bucket but Server SDKs with and API key can still access the bucket. No files are lost when this is toggled.', true) ->param('maximumFileSize', null, new Range(1, (int) App::getEnv('_APP_STORAGE_LIMIT', 0)), 'Maximum file size allowed in bytes. Maximum allowed value is ' . Storage::human((int)App::getEnv('_APP_STORAGE_LIMIT', 0), 0) . '.', true) ->param('allowedFileExtensions', [], new ArrayList(new Text(64), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Allowed file extensions. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' extensions are allowed, each 64 characters long.', true) - ->param('compression', COMPRESSION_TYPE_NONE, new WhiteList([COMPRESSION_TYPE_NONE, COMPRESSION_TYPE_GZIP, COMPRESSION_TYPE_ZSTD]), 'Compression algorithm choosen for compression. Can be one of ' . COMPRESSION_TYPE_NONE . ', [' . COMPRESSION_TYPE_GZIP . '](https://en.wikipedia.org/wiki/Gzip), or [' . COMPRESSION_TYPE_ZSTD . '](https://en.wikipedia.org/wiki/Zstd), For file size above ' . Storage::human(APP_STORAGE_READ_BUFFER, 0) . ' compression is skipped even if it\'s enabled', true) + ->param('compression', Compression::NONE, new WhiteList([Compression::NONE, Compression::GZIP, Compression::ZSTD]), 'Compression algorithm choosen for compression. Can be one of ' . Compression::NONE . ', [' . Compression::GZIP . '](https://en.wikipedia.org/wiki/Gzip), or [' . Compression::ZSTD . '](https://en.wikipedia.org/wiki/Zstd), For file size above ' . Storage::human(APP_STORAGE_READ_BUFFER, 0) . ' compression is skipped even if it\'s enabled', true) ->param('encryption', true, new Boolean(true), 'Is encryption enabled? For file size above ' . Storage::human(APP_STORAGE_READ_BUFFER, 0) . ' encryption is skipped even if it\'s enabled', true) ->param('antivirus', true, new Boolean(true), 'Is virus scanning enabled? For file size above ' . Storage::human(APP_LIMIT_ANTIVIRUS, 0) . ' AntiVirus scanning is skipped even if it\'s enabled', true) ->inject('response') @@ -531,19 +532,24 @@ App::post('/v1/storage/buckets/:bucketId/files') $fileHash = $deviceFiles->getFileHash($path); // Get file hash before compression and encryption $data = ''; // Compression - $algorithm = $bucket->getAttribute('compression', COMPRESSION_TYPE_NONE); - if ($fileSize <= APP_STORAGE_READ_BUFFER && $algorithm != COMPRESSION_TYPE_NONE) { + $algorithm = $bucket->getAttribute('compression', Compression::NONE); + if ($fileSize <= APP_STORAGE_READ_BUFFER && $algorithm != Compression::NONE) { $data = $deviceFiles->read($path); switch ($algorithm) { - case COMPRESSION_TYPE_ZSTD: + case Compression::ZSTD: $compressor = new Zstd(); break; - case COMPRESSION_TYPE_GZIP: + case Compression::GZIP: default: $compressor = new GZIP(); break; } $data = $compressor->compress($data); + } else { + // reset the algorithm to none as we do not compress the file + // if file size exceedes the APP_STORAGE_READ_BUFFER + // regardless the bucket compression algoorithm + $algorithm = Compression::NONE; } if ($bucket->getAttribute('encryption', true) && $fileSize <= APP_STORAGE_READ_BUFFER) { @@ -615,7 +621,17 @@ App::post('/v1/storage/buckets/:bucketId/files') ->setAttribute('metadata', $metadata) ->setAttribute('chunksUploaded', $chunksUploaded); - $file = $dbForProject->updateDocument('bucket_' . $bucket->getInternalId(), $fileId, $file); + /** + * Validate create permission and skip authorization in updateDocument + * Without this, the file creation will fail when user doesn't have update permission + * However as with chunk upload even if we are updating, we are essentially creating a file + * adding it's new chunk so we validate create permission instead of update + */ + $validator = new Authorization(Database::PERMISSION_CREATE); + if (!$validator->isValid($bucket->getCreate())) { + throw new Exception(Exception::USER_UNAUTHORIZED); + } + $file = Authorization::skip(fn() => $dbForProject->updateDocument('bucket_' . $bucket->getInternalId(), $fileId, $file)); } } catch (AuthorizationException) { throw new Exception(Exception::USER_UNAUTHORIZED); @@ -652,7 +668,17 @@ App::post('/v1/storage/buckets/:bucketId/files') ->setAttribute('chunksUploaded', $chunksUploaded) ->setAttribute('metadata', $metadata); - $file = $dbForProject->updateDocument('bucket_' . $bucket->getInternalId(), $fileId, $file); + /** + * Validate create permission and skip authorization in updateDocument + * Without this, the file creation will fail when user doesn't have update permission + * However as with chunk upload even if we are updating, we are essentially creating a file + * adding it's new chunk so we validate create permission instead of update + */ + $validator = new Authorization(Database::PERMISSION_CREATE); + if (!$validator->isValid($bucket->getCreate())) { + throw new Exception(Exception::USER_UNAUTHORIZED); + } + $file = Authorization::skip(fn() => $dbForProject->updateDocument('bucket_' . $bucket->getInternalId(), $fileId, $file)); } } catch (AuthorizationException) { throw new Exception(Exception::USER_UNAUTHORIZED); @@ -859,14 +885,6 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview') throw new Exception(Exception::USER_UNAUTHORIZED); } - if ((\strpos($request->getAccept(), 'image/webp') === false) && ('webp' === $output)) { // Fallback webp to jpeg when no browser support - $output = 'jpg'; - } - - $inputs = Config::getParam('storage-inputs'); - $outputs = Config::getParam('storage-outputs'); - $fileLogos = Config::getParam('storage-logos'); - if ($fileSecurity && !$valid) { $file = $dbForProject->getDocument('bucket_' . $bucket->getInternalId(), $fileId); } else { @@ -877,9 +895,17 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview') throw new Exception(Exception::STORAGE_FILE_NOT_FOUND); } + if ((\strpos($request->getAccept(), 'image/webp') === false) && ('webp' === $output)) { // Fallback webp to jpeg when no browser support + $output = 'jpg'; + } + + $inputs = Config::getParam('storage-inputs'); + $outputs = Config::getParam('storage-outputs'); + $fileLogos = Config::getParam('storage-logos'); + $path = $file->getAttribute('path'); $type = \strtolower(\pathinfo($path, PATHINFO_EXTENSION)); - $algorithm = $file->getAttribute('algorithm', 'none'); + $algorithm = $file->getAttribute('algorithm', Compression::NONE); $cipher = $file->getAttribute('openSSLCipher'); $mime = $file->getAttribute('mimeType'); if (!\in_array($mime, $inputs) || $file->getAttribute('sizeActual') > (int) App::getEnv('_APP_STORAGE_PREVIEW_LIMIT', 20000000)) { @@ -890,7 +916,7 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview') $path = $fileLogos['default_image']; } - $algorithm = 'none'; + $algorithm = Compression::NONE; $cipher = null; $background = (empty($background)) ? 'eceff1' : $background; $type = \strtolower(\pathinfo($path, PATHINFO_EXTENSION)); @@ -913,7 +939,6 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview') $output = empty($type) ? (array_search($mime, $outputs) ?? 'jpg') : $type; } - $source = $deviceFiles->read($path); if (!empty($cipher)) { // Decrypt @@ -928,11 +953,11 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview') } switch ($algorithm) { - case 'zstd': + case Compression::ZSTD: $compressor = new Zstd(); $source = $compressor->decompress($source); break; - case 'gzip': + case Compression::GZIP: $compressor = new GZIP(); $source = $compressor->decompress($source); break; @@ -1071,15 +1096,15 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/download') ); } - switch ($file->getAttribute('algorithm', 'none')) { - case 'zstd': + switch ($file->getAttribute('algorithm', Compression::NONE)) { + case Compression::ZSTD: if (empty($source)) { $source = $deviceFiles->read($path); } $compressor = new Zstd(); $source = $compressor->decompress($source); break; - case 'gzip': + case Compression::GZIP: if (empty($source)) { $source = $deviceFiles->read($path); } @@ -1220,15 +1245,15 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/view') ); } - switch ($file->getAttribute('algorithm', 'none')) { - case 'zstd': + switch ($file->getAttribute('algorithm', Compression::NONE)) { + case Compression::ZSTD: if (empty($source)) { $source = $deviceFiles->read($path); } $compressor = new Zstd(); $source = $compressor->decompress($source); break; - case 'gzip': + case Compression::GZIP: if (empty($source)) { $source = $deviceFiles->read($path); } @@ -1242,10 +1267,12 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/view') $response->send(substr($source, $start, ($end - $start + 1))); } $response->send($source); + return; } if (!empty($rangeHeader)) { $response->send($deviceFiles->read($path, $start, ($end - $start + 1))); + return; } $size = $deviceFiles->getFileSize($path); diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 9095e1d4c..d6b5a2c68 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -592,6 +592,22 @@ App::shutdown() ->setProject($project) ->trigger(); } + + /** + * Update user last activity + */ + if (!$user->isEmpty()) { + $accessedAt = $user->getAttribute('accessedAt', ''); + if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_USER_ACCCESS)) > $accessedAt) { + $user->setAttribute('accessedAt', DateTime::now()); + + if (APP_MODE_ADMIN !== $mode) { + $dbForProject->updateDocument('users', $user->getId(), $user); + } else { + $dbForConsole->updateDocument('users', $user->getId(), $user); + } + } + } }); App::init() diff --git a/app/init.php b/app/init.php index 3dbb9ec35..869ce9924 100644 --- a/app/init.php +++ b/app/init.php @@ -110,7 +110,7 @@ const APP_LIMIT_LIST_DEFAULT = 25; // Default maximum number of items to return const APP_KEY_ACCCESS = 24 * 60 * 60; // 24 hours const APP_USER_ACCCESS = 24 * 60 * 60; // 24 hours const APP_CACHE_UPDATE = 24 * 60 * 60; // 24 hours -const APP_CACHE_BUSTER = 3315; +const APP_CACHE_BUSTER = 329; const APP_VERSION_STABLE = '1.4.13'; const APP_DATABASE_ATTRIBUTE_EMAIL = 'email'; const APP_DATABASE_ATTRIBUTE_ENUM = 'enum'; @@ -174,10 +174,6 @@ const DELETE_TYPE_SESSIONS = 'sessions'; const DELETE_TYPE_CACHE_BY_TIMESTAMP = 'cacheByTimeStamp'; const DELETE_TYPE_CACHE_BY_RESOURCE = 'cacheByResource'; const DELETE_TYPE_SCHEDULES = 'schedules'; -// Compression type -const COMPRESSION_TYPE_NONE = 'none'; -const COMPRESSION_TYPE_GZIP = 'gzip'; -const COMPRESSION_TYPE_ZSTD = 'zstd'; // Mail Types const MAIL_TYPE_VERIFICATION = 'verification'; const MAIL_TYPE_MAGIC_SESSION = 'magicSession'; @@ -771,22 +767,22 @@ $register->set('pools', function () { $register->set('db', function () { // This is usually for our workers or CLI commands scope - $dbHost = App::getEnv('_APP_DB_HOST', ''); - $dbPort = App::getEnv('_APP_DB_PORT', ''); - $dbUser = App::getEnv('_APP_DB_USER', ''); - $dbPass = App::getEnv('_APP_DB_PASS', ''); - $dbScheme = App::getEnv('_APP_DB_SCHEMA', ''); + $dbHost = App::getEnv('_APP_DB_HOST', ''); + $dbPort = App::getEnv('_APP_DB_PORT', ''); + $dbUser = App::getEnv('_APP_DB_USER', ''); + $dbPass = App::getEnv('_APP_DB_PASS', ''); + $dbScheme = App::getEnv('_APP_DB_SCHEMA', ''); - $pdo = new PDO("mysql:host={$dbHost};port={$dbPort};dbname={$dbScheme};charset=utf8mb4", $dbUser, $dbPass, array( - PDO::ATTR_TIMEOUT => 3, // Seconds - PDO::ATTR_PERSISTENT => true, - PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC, - PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION, - PDO::ATTR_EMULATE_PREPARES => true, - PDO::ATTR_STRINGIFY_FETCHES => true, - )); + $pdo = new PDO("mysql:host={$dbHost};port={$dbPort};dbname={$dbScheme};charset=utf8mb4", $dbUser, $dbPass, array( + PDO::ATTR_TIMEOUT => 3, // Seconds + PDO::ATTR_PERSISTENT => true, + PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC, + PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION, + PDO::ATTR_EMULATE_PREPARES => true, + PDO::ATTR_STRINGIFY_FETCHES => true, + )); - return $pdo; + return $pdo; }); $register->set('smtp', function () { diff --git a/app/worker.php b/app/worker.php index 06be3dde3..1b1f4b9f9 100644 --- a/app/worker.php +++ b/app/worker.php @@ -9,6 +9,7 @@ use Appwrite\Event\Certificate; use Appwrite\Event\Database as EventDatabase; use Appwrite\Event\Delete; use Appwrite\Event\Func; +use Appwrite\Event\Hamster; use Appwrite\Event\Mail; use Appwrite\Event\Migration; use Appwrite\Event\Phone; @@ -178,6 +179,9 @@ Server::setResource('queueForCertificates', function (Connection $queue) { Server::setResource('queueForMigrations', function (Connection $queue) { return new Migration($queue); }, ['queue']); +Server::setResource('queueForHamster', function (Connection $queue) { + return new Hamster($queue); +}, ['queue']); Server::setResource('logger', function (Registry $register) { return $register->get('logger'); }, ['register']); diff --git a/bin/worker-hamster b/bin/worker-hamster new file mode 100644 index 000000000..b388dd13c --- /dev/null +++ b/bin/worker-hamster @@ -0,0 +1,3 @@ +#!/bin/sh + +php /usr/src/code/app/worker.php hamster $@ \ No newline at end of file diff --git a/composer.json b/composer.json index 2702df6fc..60b855ed8 100644 --- a/composer.json +++ b/composer.json @@ -52,7 +52,7 @@ "utopia-php/database": "0.45.*", "utopia-php/domains": "0.3.*", "utopia-php/dsn": "0.1.*", - "utopia-php/framework": "0.31.0", + "utopia-php/framework": "0.31.1", "utopia-php/image": "0.5.*", "utopia-php/locale": "0.4.*", "utopia-php/logger": "0.3.*", diff --git a/docker-compose.yml b/docker-compose.yml index 3eb41ae2e..7dfc92c2c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -710,6 +710,63 @@ services: environment: - _APP_ASSISTANT_OPENAI_API_KEY + appwrite-worker-hamster: + entrypoint: worker-hamster + <<: *x-logging + container_name: appwrite-worker-hamster + image: appwrite-dev + networks: + - appwrite + volumes: + - ./app:/usr/src/code/app + - ./src:/usr/src/code/src + depends_on: + - redis + - mariadb + environment: + - _APP_ENV + - _APP_WORKER_PER_CORE + - _APP_OPENSSL_KEY_V1 + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS + - _APP_MIXPANEL_TOKEN + + appwrite-hamster-scheduler: + entrypoint: hamster + <<: *x-logging + container_name: appwrite-hamster-scheduler + image: appwrite-dev + networks: + - appwrite + volumes: + - ./app:/usr/src/code/app + - ./src:/usr/src/code/src + depends_on: + - redis + - mariadb + environment: + - _APP_ENV + - _APP_WORKER_PER_CORE + - _APP_OPENSSL_KEY_V1 + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS + - _APP_HAMSTER_TIME + - _APP_HAMSTER_INTERVAL + openruntimes-executor: container_name: openruntimes-executor hostname: appwrite-executor diff --git a/src/Appwrite/Event/Event.php b/src/Appwrite/Event/Event.php index 46b430d12..fc12c5b5b 100644 --- a/src/Appwrite/Event/Event.php +++ b/src/Appwrite/Event/Event.php @@ -42,6 +42,9 @@ class Event public const MIGRATIONS_QUEUE_NAME = 'v1-migrations'; public const MIGRATIONS_CLASS_NAME = 'MigrationsV1'; + public const HAMSTER_QUEUE_NAME = 'v1-hamster'; + public const HAMSTER_CLASS_NAME = 'HamsterV1'; + protected string $queue = ''; protected string $class = ''; protected string $event = ''; diff --git a/src/Appwrite/Event/Hamster.php b/src/Appwrite/Event/Hamster.php new file mode 100644 index 000000000..5d79fce56 --- /dev/null +++ b/src/Appwrite/Event/Hamster.php @@ -0,0 +1,157 @@ +setQueue(Event::HAMSTER_QUEUE_NAME) + ->setClass(Event::HAMSTER_CLASS_NAME); + } + + /** + * Sets the type for the hamster event. + * + * @param string $type + * @return self + */ + public function setType(string $type): self + { + $this->type = $type; + + return $this; + } + + /** + * Returns the set type for the hamster event. + * + * @return string + */ + public function getType(): string + { + return $this->type; + } + + /** + * Sets the project for the hamster event. + * + * @param Document $project + */ + public function setProject(Document $project): self + { + $this->project = $project; + + return $this; + } + + /** + * Returns the set project for the hamster event. + * + * @return Document + */ + public function getProject(): Document + { + return $this->project; + } + + /** + * Sets the organization for the hamster event. + * + * @param Document $organization + */ + public function setOrganization(Document $organization): self + { + $this->organization = $organization; + + return $this; + } + + /** + * Returns the set organization for the hamster event. + * + * @return string + */ + public function getOrganization(): Document + { + return $this->organization; + } + + /** + * Sets the user for the hamster event. + * + * @param Document $user + */ + public function setUser(Document $user): self + { + $this->user = $user; + + return $this; + } + + /** + * Returns the set user for the hamster event. + * + * @return Document + */ + public function getUser(): Document + { + return $this->user; + } + + /** + * Executes the function event and sends it to the functions worker. + * + * @return string|bool + * @throws \InvalidArgumentException + */ + public function trigger(): string|bool + { + if ($this->paused) { + return false; + } + + $client = new Client($this->queue, $this->connection); + + $events = $this->getEvent() ? Event::generateEvents($this->getEvent(), $this->getParams()) : null; + + return $client->enqueue([ + 'type' => $this->type, + 'project' => $this->project, + 'organization' => $this->organization, + 'user' => $this->user, + 'events' => $events, + ]); + } + + /** + * Generate a function event from a base event + * + * @param Event $event + * + * @return self + * + */ + public function from(Event $event): self + { + $this->event = $event->getEvent(); + $this->params = $event->getParams(); + return $this; + } +} diff --git a/src/Appwrite/Platform/Services/Workers.php b/src/Appwrite/Platform/Services/Workers.php index 67ecf3dd5..6573b3124 100644 --- a/src/Appwrite/Platform/Services/Workers.php +++ b/src/Appwrite/Platform/Services/Workers.php @@ -12,6 +12,7 @@ use Appwrite\Platform\Workers\Databases; use Appwrite\Platform\Workers\Functions; use Appwrite\Platform\Workers\Builds; use Appwrite\Platform\Workers\Deletes; +use Appwrite\Platform\Workers\Hamster; use Appwrite\Platform\Workers\Usage; use Appwrite\Platform\Workers\UsageHook; use Appwrite\Platform\Workers\Migrations; @@ -31,6 +32,7 @@ class Workers extends Service ->addAction(Functions::getName(), new Functions()) ->addAction(Builds::getName(), new Builds()) ->addAction(Deletes::getName(), new Deletes()) + ->addAction(Hamster::getName(), new Hamster()) ->addAction(UsageHook::getName(), new UsageHook()) ->addAction(Usage::getName(), new Usage()) ->addAction(Migrations::getName(), new Migrations()) diff --git a/src/Appwrite/Platform/Tasks/Hamster.php b/src/Appwrite/Platform/Tasks/Hamster.php index d47c0a356..1dca095e9 100644 --- a/src/Appwrite/Platform/Tasks/Hamster.php +++ b/src/Appwrite/Platform/Tasks/Hamster.php @@ -2,46 +2,17 @@ namespace Appwrite\Platform\Tasks; -use Appwrite\Network\Validator\Origin; +use Appwrite\Event\Hamster as EventHamster; use Exception; use Utopia\App; use Utopia\Platform\Action; -use Utopia\Cache\Cache; use Utopia\CLI\Console; use Utopia\Database\Database; use Utopia\Database\Query; -use Utopia\Database\Validator\Authorization; -use Utopia\Analytics\Adapter\Mixpanel; -use Utopia\Analytics\Event; -use Utopia\Config\Config; use Utopia\Database\Document; -use Utopia\Pools\Group; class Hamster extends Action { - private array $metrics = [ - 'usage_files' => 'files', - 'usage_buckets' => 'buckets', - 'usage_databases' => 'databases', - 'usage_documents' => 'documents', - 'usage_collections' => 'collections', - 'usage_storage' => 'files.storage', - 'usage_requests' => 'network.requests', - 'usage_inbound' => 'network.inbound', - 'usage_outbound' => 'network.outbound', - 'usage_users' => 'users', - 'usage_sessions' => 'sessions', - 'usage_executions' => 'executions', - ]; - - protected string $directory = '/usr/local'; - - protected string $path; - - protected string $date; - - protected Mixpanel $mixpanel; - public static function getName(): string { return 'hamster'; @@ -49,275 +20,31 @@ class Hamster extends Action public function __construct() { - $this->mixpanel = new Mixpanel(App::getEnv('_APP_MIXPANEL_TOKEN', '')); - $this ->desc('Get stats for projects') - ->inject('pools') - ->inject('cache') + ->inject('queueForHamster') ->inject('dbForConsole') - ->callback(function (Group $pools, Cache $cache, Database $dbForConsole) { - $this->action($pools, $cache, $dbForConsole); + ->callback(function (EventHamster $queueForHamster, Database $dbForConsole) { + $this->action($queueForHamster, $dbForConsole); }); } - private function getStatsPerProject(Group $pools, Cache $cache, Database $dbForConsole) + public function action(EventHamster $queueForHamster, Database $dbForConsole): void { - $this->calculateByGroup('projects', $dbForConsole, function (Database $dbForConsole, Document $project) use ($pools, $cache) { - /** - * Skip user projects with id 'console' - */ - if ($project->getId() === 'console') { - Console::info("Skipping project console"); - return; - } - - Console::log("Getting stats for {$project->getId()}"); - - try { - $db = $project->getAttribute('database'); - $adapter = $pools - ->get($db) - ->pop() - ->getResource(); - - $dbForProject = new Database($adapter, $cache); - $dbForProject->setDefaultDatabase('appwrite'); - $dbForProject->setNamespace('_' . $project->getInternalId()); - - $statsPerProject = []; - - $statsPerProject['time'] = microtime(true); - - /** Get Project ID */ - $statsPerProject['project_id'] = $project->getId(); - - /** Get project created time */ - $statsPerProject['project_created'] = $project->getAttribute('$createdAt'); - - /** Get Project Name */ - $statsPerProject['project_name'] = $project->getAttribute('name'); - - /** Total Project Variables */ - $statsPerProject['custom_variables'] = $dbForProject->count('variables', [], APP_LIMIT_COUNT); - - /** Total Migrations */ - $statsPerProject['custom_migrations'] = $dbForProject->count('migrations', [], APP_LIMIT_COUNT); - - /** Get Custom SMTP */ - $smtp = $project->getAttribute('smtp', null); - if ($smtp) { - $statsPerProject['custom_smtp_status'] = $smtp['enabled'] === true ? 'enabled' : 'disabled'; - - /** Get Custom Templates Count */ - $templates = array_keys($project->getAttribute('templates', [])); - $statsPerProject['custom_email_templates'] = array_filter($templates, function ($template) { - return str_contains($template, 'email'); - }); - $statsPerProject['custom_sms_templates'] = array_filter($templates, function ($template) { - return str_contains($template, 'sms'); - }); - } - - /** Get total relationship attributes */ - $statsPerProject['custom_relationship_attributes'] = $dbForProject->count('attributes', [ - Query::equal('type', ['relationship']) - ], APP_LIMIT_COUNT); - - /** Get Total Functions */ - $statsPerProject['custom_functions'] = $dbForProject->count('functions', [], APP_LIMIT_COUNT); - - foreach (\array_keys(Config::getParam('runtimes')) as $runtime) { - $statsPerProject['custom_functions_' . $runtime] = $dbForProject->count('functions', [ - Query::equal('runtime', [$runtime]), - ], APP_LIMIT_COUNT); - } - - /** Get Total Deployments */ - $statsPerProject['custom_deployments'] = $dbForProject->count('deployments', [], APP_LIMIT_COUNT); - $statsPerProject['custom_deployments_manual'] = $dbForProject->count('deployments', [ - Query::equal('type', ['manual']) - ], APP_LIMIT_COUNT); - $statsPerProject['custom_deployments_git'] = $dbForProject->count('deployments', [ - Query::equal('type', ['vcs']) - ], APP_LIMIT_COUNT); - - /** Get VCS repos connected */ - $statsPerProject['custom_vcs_repositories'] = $dbForConsole->count('repositories', [ - Query::equal('projectInternalId', [$project->getInternalId()]) - ], APP_LIMIT_COUNT); - - /** Get Total Teams */ - $statsPerProject['custom_teams'] = $dbForProject->count('teams', [], APP_LIMIT_COUNT); - - /** Get Total Migrations */ - $statsPerProject['custom_migrations'] = $dbForProject->count('migrations', [], APP_LIMIT_COUNT); - - /** Get Total Members */ - $teamInternalId = $project->getAttribute('teamInternalId', null); - if ($teamInternalId) { - $statsPerProject['custom_organization_members'] = $dbForConsole->count('memberships', [ - Query::equal('teamInternalId', [$teamInternalId]) - ], APP_LIMIT_COUNT); - } else { - $statsPerProject['custom_organization_members'] = 0; - } - - /** Get Email and Name of the project owner */ - if ($teamInternalId) { - $membership = $dbForConsole->findOne('memberships', [ - Query::equal('teamInternalId', [$teamInternalId]), - ]); - - if (!$membership || $membership->isEmpty()) { - throw new Exception('Membership not found. Skipping project : ' . $project->getId()); - } - - $userId = $membership->getAttribute('userId', null); - if ($userId) { - $user = $dbForConsole->getDocument('users', $userId); - $statsPerProject['email'] = $user->getAttribute('email', null); - $statsPerProject['name'] = $user->getAttribute('name', null); - } - } - - /** Get Domains */ - $statsPerProject['custom_domains'] = $dbForConsole->count('rules', [ - Query::equal('projectInternalId', [$project->getInternalId()]), - Query::limit(APP_LIMIT_COUNT) - ]); - - /** Get Platforms */ - $platforms = $dbForConsole->find('platforms', [ - Query::equal('projectInternalId', [$project->getInternalId()]), - Query::limit(APP_LIMIT_COUNT) - ]); - - $statsPerProject['custom_platforms_web'] = sizeof(array_filter($platforms, function ($platform) { - return $platform['type'] === 'web'; - })); - - $statsPerProject['custom_platforms_android'] = sizeof(array_filter($platforms, function ($platform) { - return $platform['type'] === 'android'; - })); - - $statsPerProject['custom_platforms_apple'] = sizeof(array_filter($platforms, function ($platform) { - return str_contains($platform['type'], 'apple'); - })); - - $statsPerProject['custom_platforms_flutter'] = sizeof(array_filter($platforms, function ($platform) { - return str_contains($platform['type'], 'flutter'); - })); - - $flutterPlatforms = [Origin::CLIENT_TYPE_FLUTTER_ANDROID, Origin::CLIENT_TYPE_FLUTTER_IOS, Origin::CLIENT_TYPE_FLUTTER_MACOS, Origin::CLIENT_TYPE_FLUTTER_WINDOWS, Origin::CLIENT_TYPE_FLUTTER_LINUX]; - - foreach ($flutterPlatforms as $flutterPlatform) { - $statsPerProject['custom_platforms_' . $flutterPlatform] = sizeof(array_filter($platforms, function ($platform) use ($flutterPlatform) { - return $platform['type'] === $flutterPlatform; - })); - } - - $statsPerProject['custom_platforms_api_keys'] = $dbForConsole->count('keys', [ - Query::equal('projectInternalId', [$project->getInternalId()]), - Query::limit(APP_LIMIT_COUNT) - ]); - - /** Get Usage $statsPerProject */ - $periods = [ - 'infinity' => [ - 'period' => '1d', - 'limit' => 90, - ], - '24h' => [ - 'period' => '1h', - 'limit' => 24, - ], - ]; - - Authorization::skip(function () use ($dbForProject, $periods, &$statsPerProject) { - foreach ($this->metrics as $key => $metric) { - foreach ($periods as $periodKey => $periodValue) { - $limit = $periodValue['limit']; - $period = $periodValue['period']; - - $requestDocs = $dbForProject->find('stats_v2', [ - Query::equal('metric', [$metric]), - Query::equal('period', [$period]), - Query::limit($limit), - Query::orderDesc('time'), - ]); - - $statsPerProject[$key . '_' . $periodKey] = []; - foreach ($requestDocs as $requestDoc) { - $statsPerProject[$key . '_' . $periodKey][] = [ - 'value' => $requestDoc->getAttribute('value'), - 'date' => $requestDoc->getAttribute('time'), - ]; - } - - $statsPerProject[$key . '_' . $periodKey] = array_reverse($statsPerProject[$key . '_' . $periodKey]); - // Calculate aggregate of each metric - $statsPerProject[$key . '_' . $periodKey] = array_sum(array_column($statsPerProject[$key . '_' . $periodKey], 'value')); - } - } - }); - - /** - * Workaround to combine network.inbound+network.outbound as network. - */ - $statsPerProject["usage_network_infinity"] = $statsPerProject["usage_inbound_infinity"] + $statsPerProject["usage_outbound_infinity"]; - $statsPerProject["usage_network_24h"] = $statsPerProject["usage_inbound_24h"] + $statsPerProject["usage_outbound_24h"]; - unset($statsPerProject["usage_outbound_24h"]); - unset($statsPerProject["usage_inbound_24h"]); - unset($statsPerProject["usage_outbound_infinity"]); - unset($statsPerProject["usage_inbound_infinity"]); - - if (isset($statsPerProject['email'])) { - /** Send data to mixpanel */ - $res = $this->mixpanel->createProfile($statsPerProject['email'], '', [ - 'name' => $statsPerProject['name'], - 'email' => $statsPerProject['email'] - ]); - - if (!$res) { - Console::error('Failed to create user profile for project: ' . $project->getId()); - } - } - - $event = new Event(); - $event - ->setName('Project Daily Usage') - ->setProps($statsPerProject); - $res = $this->mixpanel->createEvent($event); - - if (!$res) { - Console::error('Failed to create event for project: ' . $project->getId()); - } - } catch (Exception $e) { - Console::error('Failed to send stats for project: ' . $project->getId()); - Console::error($e->getMessage()); - } finally { - $pools - ->get($db) - ->reclaim(); - } - }); - } - - public function action(Group $pools, Cache $cache, Database $dbForConsole): void - { - Console::title('Cloud Hamster V1'); Console::success(APP_NAME . ' cloud hamster process has started'); $sleep = (int) App::getEnv('_APP_HAMSTER_INTERVAL', '30'); // 30 seconds (by default) + $jobInitTime = App::getEnv('_APP_HAMSTER_TIME', '22:00'); // (hour:minutes) + $now = new \DateTime(); $now->setTimezone(new \DateTimeZone(date_default_timezone_get())); + $next = new \DateTime($now->format("Y-m-d $jobInitTime")); $next->setTimezone(new \DateTimeZone(date_default_timezone_get())); - $delay = $next->getTimestamp() - $now->getTimestamp(); + $delay = $next->getTimestamp() - $now->getTimestamp(); /** * If time passed for the target day. */ @@ -325,31 +52,25 @@ class Hamster extends Action $next->add(\DateInterval::createFromDateString('1 days')); $delay = $next->getTimestamp() - $now->getTimestamp(); } + Console::log('[' . $now->format("Y-m-d H:i:s.v") . '] Delaying for ' . $delay . ' setting loop to [' . $next->format("Y-m-d H:i:s.v") . ']'); - Console::loop(function () use ($pools, $cache, $dbForConsole, $sleep) { + Console::loop(function () use ($queueForHamster, $dbForConsole, $sleep) { $now = date('d-m-Y H:i:s', time()); - Console::info("[{$now}] Getting Cloud Usage Stats every {$sleep} seconds"); + Console::info("[{$now}] Queuing Cloud Usage Stats every {$sleep} seconds"); $loopStart = microtime(true); - /* Initialise new Utopia app */ - $app = new App('UTC'); + Console::info('Queuing stats for all projects'); + $this->getStatsPerProject($queueForHamster, $dbForConsole, $loopStart); + Console::success('Completed queuing stats for all projects'); - Console::info('Getting stats for all projects'); - $this->getStatsPerProject($pools, $cache, $dbForConsole); - Console::success('Completed getting stats for all projects'); + Console::info('Queuing stats for all organizations'); + $this->getStatsPerOrganization($queueForHamster, $dbForConsole, $loopStart); + Console::success('Completed queuing stats for all organizations'); - Console::info('Getting stats for all organizations'); - $this->getStatsPerOrganization($dbForConsole); - Console::success('Completed getting stats for all organizations'); - - Console::info('Getting stats for all users'); - $this->getStatsPerUser($dbForConsole); - Console::success('Completed getting stats for all users'); - - $pools - ->get('console') - ->reclaim(); + Console::info('Queuing stats for all users'); + $this->getStatsPerUser($queueForHamster, $dbForConsole, $loopStart); + Console::success('Completed queuing stats for all users'); $loopTook = microtime(true) - $loopStart; $now = date('d-m-Y H:i:s', time()); @@ -357,7 +78,7 @@ class Hamster extends Action }, $sleep, $delay); } - protected function calculateByGroup(string $collection, Database $dbForConsole, callable $callback) + protected function calculateByGroup(string $collection, Database $database, callable $callback) { $count = 0; $chunk = 0; @@ -370,7 +91,7 @@ class Hamster extends Action while ($sum === $limit) { $chunk++; - $results = $dbForConsole->find($collection, \array_merge([ + $results = $database->find($collection, \array_merge([ Query::limit($limit), Query::offset($count) ])); @@ -380,7 +101,7 @@ class Hamster extends Action Console::log('Processing chunk #' . $chunk . '. Found ' . $sum . ' documents'); foreach ($results as $document) { - call_user_func($callback, $dbForConsole, $document); + call_user_func($callback, $database, $document); $count++; } } @@ -390,96 +111,45 @@ class Hamster extends Action Console::log("Processed {$count} document by group in " . ($executionEnd - $executionStart) . " seconds"); } - protected function getStatsPerOrganization(Database $dbForConsole) + protected function getStatsPerOrganization(EventHamster $hamster, Database $dbForConsole, float $loopStart) { - - $this->calculateByGroup('teams', $dbForConsole, function (Database $dbForConsole, Document $document) { + $this->calculateByGroup('teams', $dbForConsole, function (Database $dbForConsole, Document $organization) use ($hamster, $loopStart) { try { - $statsPerOrganization = []; - - /** Organization name */ - $statsPerOrganization['name'] = $document->getAttribute('name'); - - /** Get Email and of the organization owner */ - $membership = $dbForConsole->findOne('memberships', [ - Query::equal('teamInternalId', [$document->getInternalId()]), - ]); - - if (!$membership || $membership->isEmpty()) { - throw new Exception('Membership not found. Skipping organization : ' . $document->getId()); - } - - $userId = $membership->getAttribute('userId', null); - if ($userId) { - $user = $dbForConsole->getDocument('users', $userId); - $statsPerOrganization['email'] = $user->getAttribute('email', null); - } - - /** Organization Creation Date */ - $statsPerOrganization['created'] = $document->getAttribute('$createdAt'); - - /** Number of team members */ - $statsPerOrganization['members'] = $document->getAttribute('total'); - - /** Number of projects in this organization */ - $statsPerOrganization['projects'] = $dbForConsole->count('projects', [ - Query::equal('teamId', [$document->getId()]), - Query::limit(APP_LIMIT_COUNT) - ]); - - if (!isset($statsPerOrganization['email'])) { - throw new Exception('Email not found. Skipping organization : ' . $document->getId()); - } - - $event = new Event(); - $event - ->setName('Organization Daily Usage') - ->setProps($statsPerOrganization); - $res = $this->mixpanel->createEvent($event); - if (!$res) { - throw new Exception('Failed to create event for organization : ' . $document->getId()); - } + $organization->setAttribute('$time', $loopStart); + $hamster + ->setType(EventHamster::TYPE_ORGANISATION) + ->setOrganization($organization) + ->trigger(); } catch (Exception $e) { Console::error($e->getMessage()); } }); } - protected function getStatsPerUser(Database $dbForConsole) + private function getStatsPerProject(EventHamster $hamster, Database $dbForConsole, float $loopStart) { - $this->calculateByGroup('users', $dbForConsole, function (Database $dbForConsole, Document $document) { + $this->calculateByGroup('projects', $dbForConsole, function (Database $dbForConsole, Document $project) use ($hamster, $loopStart) { try { - $statsPerUser = []; + $project->setAttribute('$time', $loopStart); + $hamster + ->setType(EventHamster::TYPE_PROJECT) + ->setProject($project) + ->trigger(); + } catch (Exception $e) { + Console::error($e->getMessage()); + } + }); + } - /** Organization name */ - $statsPerUser['name'] = $document->getAttribute('name'); - - /** Organization ID (needs to be stored as an email since mixpanel uses the email attribute as a distinctID) */ - $statsPerUser['email'] = $document->getAttribute('email'); - - /** Organization Creation Date */ - $statsPerUser['created'] = $document->getAttribute('$createdAt'); - - /** Number of teams this user is a part of */ - $statsPerUser['memberships'] = $dbForConsole->count('memberships', [ - Query::equal('userInternalId', [$document->getInternalId()]), - Query::limit(APP_LIMIT_COUNT) - ]); - - if (!isset($statsPerUser['email'])) { - throw new Exception('User has no email: ' . $document->getId()); - } - - /** Send data to mixpanel */ - $event = new Event(); - $event - ->setName('User Daily Usage') - ->setProps($statsPerUser); - $res = $this->mixpanel->createEvent($event); - - if (!$res) { - throw new Exception('Failed to create user profile for user: ' . $document->getId()); - } + protected function getStatsPerUser(EventHamster $hamster, Database $dbForConsole, float $loopStart) + { + $this->calculateByGroup('users', $dbForConsole, function (Database $dbForConsole, Document $user) use ($hamster, $loopStart) { + try { + $user->setAttribute('$time', $loopStart); + $hamster + ->setType(EventHamster::TYPE_USER) + ->setUser($user) + ->trigger(); } catch (Exception $e) { Console::error($e->getMessage()); } diff --git a/src/Appwrite/Platform/Workers/Hamster.php b/src/Appwrite/Platform/Workers/Hamster.php new file mode 100644 index 000000000..e911bb6c7 --- /dev/null +++ b/src/Appwrite/Platform/Workers/Hamster.php @@ -0,0 +1,437 @@ + 'files.$all.count.total', + 'usage_buckets' => 'buckets.$all.count.total', + 'usage_databases' => 'databases.$all.count.total', + 'usage_documents' => 'documents.$all.count.total', + 'usage_collections' => 'collections.$all.count.total', + 'usage_storage' => 'project.$all.storage.size', + 'usage_requests' => 'project.$all.network.requests', + 'usage_bandwidth' => 'project.$all.network.bandwidth', + 'usage_users' => 'users.$all.count.total', + 'usage_sessions' => 'sessions.email.requests.create', + 'usage_executions' => 'executions.$all.compute.total', + ]; + + protected Mixpanel $mixpanel; + + public static function getName(): string + { + return 'hamster'; + } + + /** + * @throws \Exception + */ + public function __construct() + { + $this + ->desc('Hamster worker') + ->inject('message') + ->inject('pools') + ->inject('cache') + ->inject('dbForConsole') + ->callback(fn (Message $message, Group $pools, Cache $cache, Database $dbForConsole) => $this->action($message, $pools, $cache, $dbForConsole)); + } + + /** + * @param Message $message + * @param Group $pools + * @param Cache $cache + * @param Database $dbForConsole + * + * @return void + * @throws \Utopia\Database\Exception + */ + public function action(Message $message, Group $pools, Cache $cache, Database $dbForConsole): void + { + $token = App::getEnv('_APP_MIXPANEL_TOKEN', ''); + if (empty($token)) { + throw new \Exception('Missing MixPanel Token'); + } + $this->mixpanel = new Mixpanel($token); + + $payload = $message->getPayload() ?? []; + + if (empty($payload)) { + throw new \Exception('Missing payload'); + } + + $type = $payload['type'] ?? ''; + + switch ($type) { + case EventHamster::TYPE_PROJECT: + $this->getStatsForProject(new Document($payload['project']), $pools, $cache, $dbForConsole); + break; + case EventHamster::TYPE_ORGANISATION: + $this->getStatsForOrganization(new Document($payload['organization']), $dbForConsole); + break; + case EventHamster::TYPE_USER: + $this->getStatsPerUser(new Document($payload['user']), $dbForConsole); + break; + } + } + + /** + * @param Document $project + * @param Group $pools + * @param Cache $cache + * @param Database $dbForConsole + * @throws \Utopia\Database\Exception + */ + private function getStatsForProject(Document $project, Group $pools, Cache $cache, Database $dbForConsole): void + { + /** + * Skip user projects with id 'console' + */ + if ($project->getId() === 'console') { + Console::info("Skipping project console"); + return; + } + + Console::log("Getting stats for Project {$project->getId()}"); + + try { + $db = $project->getAttribute('database'); + $adapter = $pools + ->get($db) + ->pop() + ->getResource(); + + $dbForProject = new Database($adapter, $cache); + $dbForProject->setDefaultDatabase('appwrite'); + $dbForProject->setNamespace('_' . $project->getInternalId()); + + $statsPerProject = []; + + $statsPerProject['time'] = $project->getAttribute('$time'); + + /** Get Project ID */ + $statsPerProject['project_id'] = $project->getId(); + + /** Get project created time */ + $statsPerProject['project_created'] = $project->getAttribute('$createdAt'); + + /** Get Project Name */ + $statsPerProject['project_name'] = $project->getAttribute('name'); + + /** Total Project Variables */ + $statsPerProject['custom_variables'] = $dbForProject->count('variables', [], APP_LIMIT_COUNT); + + /** Total Migrations */ + $statsPerProject['custom_migrations'] = $dbForProject->count('migrations', [], APP_LIMIT_COUNT); + + /** Get Custom SMTP */ + $smtp = $project->getAttribute('smtp', null); + if ($smtp) { + $statsPerProject['custom_smtp_status'] = $smtp['enabled'] === true ? 'enabled' : 'disabled'; + + /** Get Custom Templates Count */ + $templates = array_keys($project->getAttribute('templates', [])); + $statsPerProject['custom_email_templates'] = array_filter($templates, function ($template) { + return str_contains($template, 'email'); + }); + $statsPerProject['custom_sms_templates'] = array_filter($templates, function ($template) { + return str_contains($template, 'sms'); + }); + } + + /** Get total relationship attributes */ + $statsPerProject['custom_relationship_attributes'] = $dbForProject->count('attributes', [ + Query::equal('type', ['relationship']) + ], APP_LIMIT_COUNT); + + /** Get Total Functions */ + $statsPerProject['custom_functions'] = $dbForProject->count('functions', [], APP_LIMIT_COUNT); + + foreach (\array_keys(Config::getParam('runtimes')) as $runtime) { + $statsPerProject['custom_functions_' . $runtime] = $dbForProject->count('functions', [ + Query::equal('runtime', [$runtime]), + ], APP_LIMIT_COUNT); + } + + /** Get Total Deployments */ + $statsPerProject['custom_deployments'] = $dbForProject->count('deployments', [], APP_LIMIT_COUNT); + $statsPerProject['custom_deployments_manual'] = $dbForProject->count('deployments', [ + Query::equal('type', ['manual']) + ], APP_LIMIT_COUNT); + $statsPerProject['custom_deployments_git'] = $dbForProject->count('deployments', [ + Query::equal('type', ['vcs']) + ], APP_LIMIT_COUNT); + + /** Get VCS repos connected */ + $statsPerProject['custom_vcs_repositories'] = $dbForConsole->count('repositories', [ + Query::equal('projectInternalId', [$project->getInternalId()]) + ], APP_LIMIT_COUNT); + + /** Get Total Teams */ + $statsPerProject['custom_teams'] = $dbForProject->count('teams', [], APP_LIMIT_COUNT); + + /** Get Total Members */ + $teamInternalId = $project->getAttribute('teamInternalId', null); + if ($teamInternalId) { + $statsPerProject['custom_organization_members'] = $dbForConsole->count('memberships', [ + Query::equal('teamInternalId', [$teamInternalId]) + ], APP_LIMIT_COUNT); + } else { + $statsPerProject['custom_organization_members'] = 0; + } + + /** Get Email and Name of the project owner */ + if ($teamInternalId) { + $membership = $dbForConsole->findOne('memberships', [ + Query::equal('teamInternalId', [$teamInternalId]), + ]); + + if (!$membership || $membership->isEmpty()) { + throw new \Exception('Membership not found. Skipping project : ' . $project->getId()); + } + + $userId = $membership->getAttribute('userId', null); + if ($userId) { + $user = $dbForConsole->getDocument('users', $userId); + $statsPerProject['email'] = $user->getAttribute('email', null); + $statsPerProject['name'] = $user->getAttribute('name', null); + } + } + + /** Get Domains */ + $statsPerProject['custom_domains'] = $dbForConsole->count('rules', [ + Query::equal('projectInternalId', [$project->getInternalId()]), + Query::limit(APP_LIMIT_COUNT) + ]); + + /** Get Platforms */ + $platforms = $dbForConsole->find('platforms', [ + Query::equal('projectInternalId', [$project->getInternalId()]), + Query::limit(APP_LIMIT_COUNT) + ]); + + $statsPerProject['custom_platforms_web'] = sizeof(array_filter($platforms, function ($platform) { + return $platform['type'] === 'web'; + })); + + $statsPerProject['custom_platforms_android'] = sizeof(array_filter($platforms, function ($platform) { + return $platform['type'] === 'android'; + })); + + $statsPerProject['custom_platforms_apple'] = sizeof(array_filter($platforms, function ($platform) { + return str_contains($platform['type'], 'apple'); + })); + + $statsPerProject['custom_platforms_flutter'] = sizeof(array_filter($platforms, function ($platform) { + return str_contains($platform['type'], 'flutter'); + })); + + $flutterPlatforms = [Origin::CLIENT_TYPE_FLUTTER_ANDROID, Origin::CLIENT_TYPE_FLUTTER_IOS, Origin::CLIENT_TYPE_FLUTTER_MACOS, Origin::CLIENT_TYPE_FLUTTER_WINDOWS, Origin::CLIENT_TYPE_FLUTTER_LINUX]; + + foreach ($flutterPlatforms as $flutterPlatform) { + $statsPerProject['custom_platforms_' . $flutterPlatform] = sizeof(array_filter($platforms, function ($platform) use ($flutterPlatform) { + return $platform['type'] === $flutterPlatform; + })); + } + + $statsPerProject['custom_platforms_api_keys'] = $dbForConsole->count('keys', [ + Query::equal('projectInternalId', [$project->getInternalId()]), + Query::limit(APP_LIMIT_COUNT) + ]); + + /** Get Usage $statsPerProject */ + $periods = [ + 'infinity' => [ + 'period' => '1d', + 'limit' => 90, + ], + '24h' => [ + 'period' => '1h', + 'limit' => 24, + ], + ]; + + Authorization::skip(function () use ($dbForProject, $periods, &$statsPerProject) { + foreach ($this->metrics as $key => $metric) { + foreach ($periods as $periodKey => $periodValue) { + $limit = $periodValue['limit']; + $period = $periodValue['period']; + + $requestDocs = $dbForProject->find('stats', [ + Query::equal('period', [$period]), + Query::equal('metric', [$metric]), + Query::limit($limit), + Query::orderDesc('time'), + ]); + + $statsPerProject[$key . '_' . $periodKey] = []; + foreach ($requestDocs as $requestDoc) { + $statsPerProject[$key . '_' . $periodKey][] = [ + 'value' => $requestDoc->getAttribute('value'), + 'date' => $requestDoc->getAttribute('time'), + ]; + } + + $statsPerProject[$key . '_' . $periodKey] = array_reverse($statsPerProject[$key . '_' . $periodKey]); + // Calculate aggregate of each metric + $statsPerProject[$key . '_' . $periodKey] = array_sum(array_column($statsPerProject[$key . '_' . $periodKey], 'value')); + } + } + }); + + if (isset($statsPerProject['email'])) { + /** Send data to mixpanel */ + $res = $this->mixpanel->createProfile($statsPerProject['email'], '', [ + 'name' => $statsPerProject['name'], + 'email' => $statsPerProject['email'] + ]); + + if (!$res) { + Console::error('Failed to create user profile for project: ' . $project->getId()); + } + } + + $event = new AnalyticsEvent(); + $event + ->setName('Project Daily Usage') + ->setProps($statsPerProject); + $res = $this->mixpanel->createEvent($event); + + if (!$res) { + Console::error('Failed to create event for project: ' . $project->getId()); + } + } catch (\Exception $e) { + Console::error('Failed to send stats for project: ' . $project->getId()); + Console::error($e->getMessage()); + } finally { + $pools + ->get($db) + ->reclaim(); + } + } + + /** + * @param Document $organization + * @param Database $dbForConsole + * @throws \Utopia\Database\Exception + */ + private function getStatsForOrganization(Document $organization, Database $dbForConsole): void + { + Console::log("Getting stats for Organization {$organization->getId()}"); + + try { + $statsPerOrganization = []; + + $statsPerOrganization['time'] = $organization->getAttribute('$time'); + + /** Organization name */ + $statsPerOrganization['name'] = $organization->getAttribute('name'); + + + /** Get Email and of the organization owner */ + $membership = $dbForConsole->findOne('memberships', [ + Query::equal('teamInternalId', [$organization->getInternalId()]), + ]); + + if (!$membership || $membership->isEmpty()) { + throw new \Exception('Membership not found. Skipping organization : ' . $organization->getId()); + } + + $userId = $membership->getAttribute('userId', null); + if ($userId) { + $user = $dbForConsole->getDocument('users', $userId); + $statsPerOrganization['email'] = $user->getAttribute('email', null); + } + + /** Organization Creation Date */ + $statsPerOrganization['created'] = $organization->getAttribute('$createdAt'); + + /** Number of team members */ + $statsPerOrganization['members'] = $organization->getAttribute('total'); + + /** Number of projects in this organization */ + $statsPerOrganization['projects'] = $dbForConsole->count('projects', [ + Query::equal('teamId', [$organization->getId()]), + Query::limit(APP_LIMIT_COUNT) + ]); + + if (!isset($statsPerOrganization['email'])) { + throw new \Exception('Email not found. Skipping organization : ' . $organization->getId()); + } + + $event = new AnalyticsEvent(); + $event + ->setName('Organization Daily Usage') + ->setProps($statsPerOrganization); + $res = $this->mixpanel->createEvent($event); + if (!$res) { + throw new \Exception('Failed to create event for organization : ' . $organization->getId()); + } + } catch (\Exception $e) { + Console::error($e->getMessage()); + } + } + + protected function getStatsPerUser(Document $user, Database $dbForConsole) + { + Console::log("Getting stats for User {$user->getId()}"); + + try { + $statsPerUser = []; + + $statsPerUser['time'] = $user->getAttribute('$time'); + + /** Organization name */ + $statsPerUser['name'] = $user->getAttribute('name'); + + /** Organization ID (needs to be stored as an email since mixpanel uses the email attribute as a distinctID) */ + $statsPerUser['email'] = $user->getAttribute('email'); + + /** Organization Creation Date */ + $statsPerUser['created'] = $user->getAttribute('$createdAt'); + + /** Number of teams this user is a part of */ + $statsPerUser['memberships'] = $dbForConsole->count('memberships', [ + Query::equal('userInternalId', [$user->getInternalId()]), + Query::limit(APP_LIMIT_COUNT) + ]); + + if (!isset($statsPerUser['email'])) { + throw new \Exception('User has no email: ' . $user->getId()); + } + + /** Send data to mixpanel */ + $event = new AnalyticsEvent(); + $event + ->setName('User Daily Usage') + ->setProps($statsPerUser); + + $res = $this->mixpanel->createEvent($event); + + if (!$res) { + throw new \Exception('Failed to create user profile for user: ' . $user->getId()); + } + } catch (\Exception $e) { + Console::error($e->getMessage()); + } + } +} diff --git a/src/Appwrite/Utopia/Response/Model/Bucket.php b/src/Appwrite/Utopia/Response/Model/Bucket.php index 3c5715efc..f5261c026 100644 --- a/src/Appwrite/Utopia/Response/Model/Bucket.php +++ b/src/Appwrite/Utopia/Response/Model/Bucket.php @@ -4,6 +4,7 @@ namespace Appwrite\Utopia\Response\Model; use Appwrite\Utopia\Response; use Appwrite\Utopia\Response\Model; +use Utopia\Storage\Compression\Compression; class Bucket extends Model { @@ -68,7 +69,7 @@ class Bucket extends Model ]) ->addRule('compression', [ 'type' => self::TYPE_STRING, - 'description' => 'Compression algorithm choosen for compression. Will be one of ' . COMPRESSION_TYPE_NONE . ', [' . COMPRESSION_TYPE_GZIP . '](https://en.wikipedia.org/wiki/Gzip), or [' . COMPRESSION_TYPE_ZSTD . '](https://en.wikipedia.org/wiki/Zstd).', + 'description' => 'Compression algorithm choosen for compression. Will be one of ' . Compression::NONE . ', [' . Compression::GZIP . '](https://en.wikipedia.org/wiki/Gzip), or [' . Compression::ZSTD . '](https://en.wikipedia.org/wiki/Zstd).', 'default' => '', 'example' => 'gzip', 'array' => false diff --git a/tests/e2e/Client.php b/tests/e2e/Client.php index 21e4ccc95..7083095da 100644 --- a/tests/e2e/Client.php +++ b/tests/e2e/Client.php @@ -168,6 +168,7 @@ class Client $headers = array_merge($this->headers, $headers); $ch = curl_init($this->endpoint . $path . (($method == self::METHOD_GET && !empty($params)) ? '?' . http_build_query($params) : '')); $responseHeaders = []; + $cookies = []; $query = match ($headers['content-type']) { 'application/json' => json_encode($params), @@ -189,7 +190,7 @@ class Client curl_setopt($ch, CURLOPT_HTTPHEADER, $headers); curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 0); curl_setopt($ch, CURLOPT_TIMEOUT, 15); - curl_setopt($ch, CURLOPT_HEADERFUNCTION, function ($curl, $header) use (&$responseHeaders) { + curl_setopt($ch, CURLOPT_HEADERFUNCTION, function ($curl, $header) use (&$responseHeaders, &$cookies) { $len = strlen($header); $header = explode(':', $header, 2); @@ -197,6 +198,12 @@ class Client return $len; } + if (strtolower(trim($header[0])) == 'set-cookie') { + $parsed = $this->parseCookie((string)trim($header[1])); + $name = array_key_first($parsed); + $cookies[$name] = $parsed[$name]; + } + $responseHeaders[strtolower(trim($header[0]))] = trim($header[1]); return $len; @@ -241,6 +248,7 @@ class Client return [ 'headers' => $responseHeaders, + 'cookies' => $cookies, 'body' => $responseBody ]; } diff --git a/tests/e2e/Scopes/Scope.php b/tests/e2e/Scopes/Scope.php index 14eb83897..8d0f44118 100644 --- a/tests/e2e/Scopes/Scope.php +++ b/tests/e2e/Scopes/Scope.php @@ -98,7 +98,7 @@ abstract class Scope extends TestCase 'password' => $password, ]); - $session = $this->client->parseCookie((string)$session['headers']['set-cookie'])['a_session_console']; + $session = $session['cookies']['a_session_console']; self::$root = [ '$id' => ID::custom($root['body']['$id']), @@ -150,7 +150,7 @@ abstract class Scope extends TestCase 'password' => $password, ]); - $token = $this->client->parseCookie((string)$session['headers']['set-cookie'])['a_session_' . $this->getProject()['$id']]; + $token = $session['cookies']['a_session_' . $this->getProject()['$id']]; self::$user[$this->getProject()['$id']] = [ '$id' => ID::custom($user['body']['$id']), diff --git a/tests/e2e/Services/Account/AccountBase.php b/tests/e2e/Services/Account/AccountBase.php index 953b89ced..e6f5feaa8 100644 --- a/tests/e2e/Services/Account/AccountBase.php +++ b/tests/e2e/Services/Account/AccountBase.php @@ -126,7 +126,7 @@ trait AccountBase $this->assertNotFalse(\DateTime::createFromFormat('Y-m-d\TH:i:s.uP', $response['body']['expire'])); $sessionId = $response['body']['$id']; - $session = $this->client->parseCookie((string)$response['headers']['set-cookie'])['a_session_' . $this->getProject()['$id']]; + $session = $response['cookies']['a_session_' . $this->getProject()['$id']]; // apiKey is only available in custom client test $apiKey = $this->getProject()['apiKey']; @@ -993,7 +993,7 @@ trait AccountBase ]); $sessionNewId = $response['body']['$id']; - $sessionNew = $this->client->parseCookie((string)$response['headers']['set-cookie'])['a_session_' . $this->getProject()['$id']]; + $sessionNew = $response['cookies']['a_session_' . $this->getProject()['$id']]; $this->assertEquals($response['headers']['status-code'], 201); @@ -1059,7 +1059,7 @@ trait AccountBase 'password' => $password, ]); - $sessionNew = $this->client->parseCookie((string)$response['headers']['set-cookie'])['a_session_' . $this->getProject()['$id']]; + $sessionNew = $response['cookies']['a_session_' . $this->getProject()['$id']]; $this->assertEquals($response['headers']['status-code'], 201); @@ -1141,7 +1141,7 @@ trait AccountBase 'password' => $password, ]); - $data['session'] = $this->client->parseCookie((string)$response['headers']['set-cookie'])['a_session_' . $this->getProject()['$id']]; + $data['session'] = $response['cookies']['a_session_' . $this->getProject()['$id']]; return $data; } @@ -1417,7 +1417,7 @@ trait AccountBase $this->assertNotEmpty($response['body']['userId']); $sessionId = $response['body']['$id']; - $session = $this->client->parseCookie((string)$response['headers']['set-cookie'])['a_session_' . $this->getProject()['$id']]; + $session = $response['cookies']['a_session_' . $this->getProject()['$id']]; $response = $this->client->call(Client::METHOD_GET, '/account', array_merge([ 'origin' => 'http://localhost', diff --git a/tests/e2e/Services/Account/AccountCustomClientTest.php b/tests/e2e/Services/Account/AccountCustomClientTest.php index 8e2117032..56280b4b4 100644 --- a/tests/e2e/Services/Account/AccountCustomClientTest.php +++ b/tests/e2e/Services/Account/AccountCustomClientTest.php @@ -126,7 +126,7 @@ class AccountCustomClientTest extends Scope $this->assertEquals($response['headers']['status-code'], 201); $sessionId = $response['body']['$id']; - $session = $this->client->parseCookie((string)$response['headers']['set-cookie'])['a_session_' . $this->getProject()['$id']]; + $session = $response['cookies']['a_session_' . $this->getProject()['$id']]; $response = $this->client->call(Client::METHOD_GET, '/account', array_merge([ 'origin' => 'http://localhost', @@ -206,7 +206,7 @@ class AccountCustomClientTest extends Scope $this->assertEquals($response['headers']['status-code'], 201); - $session = $this->client->parseCookie((string)$response['headers']['set-cookie'])['a_session_' . $this->getProject()['$id']]; + $session = $response['cookies']['a_session_' . $this->getProject()['$id']]; $response = $this->client->call(Client::METHOD_GET, '/account', array_merge([ 'origin' => 'http://localhost', @@ -288,7 +288,7 @@ class AccountCustomClientTest extends Scope $this->assertEquals($response['headers']['status-code'], 201); $sessionId = $response['body']['$id']; - $session = $this->client->parseCookie((string)$response['headers']['set-cookie'])['a_session_' . $this->getProject()['$id']]; + $session = $response['cookies']['a_session_' . $this->getProject()['$id']]; $response = $this->client->call(Client::METHOD_GET, '/account', array_merge([ 'origin' => 'http://localhost', @@ -368,7 +368,7 @@ class AccountCustomClientTest extends Scope $this->assertNotEmpty($response['body']); $this->assertNotEmpty($response['body']['$id']); - $session = $this->client->parseCookie((string)$response['headers']['set-cookie'])['a_session_' . $this->getProject()['$id']]; + $session = $response['cookies']['a_session_' . $this->getProject()['$id']]; \usleep(1000 * 30); // wait for 30ms to let the shutdown update accessedAt @@ -571,7 +571,7 @@ class AccountCustomClientTest extends Scope 'failure' => 'http://localhost/v1/mock/tests/general/oauth2/failure', ]); - $session = $this->client->parseCookie((string)$response['headers']['set-cookie'])['a_session_' . $this->getProject()['$id']]; + $session = $response['cookies']['a_session_' . $this->getProject()['$id']]; $this->assertEquals(200, $response['headers']['status-code']); $this->assertEquals('success', $response['body']['result']); @@ -849,7 +849,7 @@ class AccountCustomClientTest extends Scope $this->assertNotEmpty($response['body']['$id']); $this->assertNotEmpty($response['body']['userId']); - $session = $this->client->parseCookie((string)$response['headers']['set-cookie'])['a_session_' . $this->getProject()['$id']]; + $session = $response['cookies']['a_session_' . $this->getProject()['$id']]; $response = $this->client->call(Client::METHOD_GET, '/account', array_merge([ 'origin' => 'http://localhost', diff --git a/tests/e2e/Services/Databases/DatabasesBase.php b/tests/e2e/Services/Databases/DatabasesBase.php index f86b81777..ece495686 100644 --- a/tests/e2e/Services/Databases/DatabasesBase.php +++ b/tests/e2e/Services/Databases/DatabasesBase.php @@ -2791,7 +2791,7 @@ trait DatabasesBase 'email' => $email, 'password' => $password, ]); - $session2 = $this->client->parseCookie((string)$session2['headers']['set-cookie'])['a_session_' . $this->getProject()['$id']]; + $session2 = $session2['cookies']['a_session_' . $this->getProject()['$id']]; $document3GetWithDocumentRead = $this->client->call(Client::METHOD_GET, '/databases/' . $databaseId . '/collections/' . $collectionId . '/documents/' . $document3['body']['$id'], [ 'origin' => 'http://localhost', @@ -2979,7 +2979,7 @@ trait DatabasesBase 'email' => $email, 'password' => $password, ]); - $session2 = $this->client->parseCookie((string)$session2['headers']['set-cookie'])['a_session_' . $this->getProject()['$id']]; + $session2 = $session2['cookies']['a_session_' . $this->getProject()['$id']]; $document3GetWithDocumentRead = $this->client->call(Client::METHOD_GET, '/databases/' . $databaseId . '/collections/' . $collectionId . '/documents/' . $document3['body']['$id'], [ 'origin' => 'http://localhost', diff --git a/tests/e2e/Services/Databases/DatabasesPermissionsScope.php b/tests/e2e/Services/Databases/DatabasesPermissionsScope.php index 181231adc..336e47db0 100644 --- a/tests/e2e/Services/Databases/DatabasesPermissionsScope.php +++ b/tests/e2e/Services/Databases/DatabasesPermissionsScope.php @@ -32,7 +32,7 @@ trait DatabasesPermissionsScope 'password' => $password, ]); - $session = $this->client->parseCookie((string)$session['headers']['set-cookie'])['a_session_' . $this->getProject()['$id']]; + $session = $session['cookies']['a_session_' . $this->getProject()['$id']]; $user = [ '$id' => $user['body']['$id'], diff --git a/tests/e2e/Services/GraphQL/AccountTest.php b/tests/e2e/Services/GraphQL/AccountTest.php index 7fd70b501..3985988d7 100644 --- a/tests/e2e/Services/GraphQL/AccountTest.php +++ b/tests/e2e/Services/GraphQL/AccountTest.php @@ -63,7 +63,7 @@ class AccountTest extends Scope $this->assertIsArray($session['body']['data']); $this->assertIsArray($session['body']['data']['accountCreateEmailSession']); - $cookie = $this->client->parseCookie((string)$session['headers']['set-cookie'])['a_session_' . $this->getProject()['$id']]; + $cookie = $session['cookies']['a_session_' . $this->getProject()['$id']]; $this->assertNotEmpty($cookie); } diff --git a/tests/e2e/Services/GraphQL/AuthTest.php b/tests/e2e/Services/GraphQL/AuthTest.php index c331fb843..b8689c144 100644 --- a/tests/e2e/Services/GraphQL/AuthTest.php +++ b/tests/e2e/Services/GraphQL/AuthTest.php @@ -73,9 +73,7 @@ class AuthTest extends Scope 'x-appwrite-project' => $projectId, ], $graphQLPayload); - $this->token1 = $this->client->parseCookie( - (string)$session1['headers']['set-cookie'] - )['a_session_' . $projectId]; + $this->token1 = $session1['cookies']['a_session_' . $projectId]; // Create session 2 $graphQLPayload['variables']['email'] = $email2; @@ -85,9 +83,7 @@ class AuthTest extends Scope 'x-appwrite-project' => $projectId, ], $graphQLPayload); - $this->token2 = $this->client->parseCookie( - (string)$session2['headers']['set-cookie'] - )['a_session_' . $projectId]; + $this->token2 = $session2['cookies']['a_session_' . $projectId]; // Create database $query = $this->getQuery(self::$CREATE_DATABASE); diff --git a/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php b/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php index 18a568da7..ef9ff6c3e 100644 --- a/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php +++ b/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php @@ -933,7 +933,7 @@ class ProjectsConsoleClientTest extends Scope 'password' => $originalPassword, ]); - $session = $this->client->parseCookie((string)$response['headers']['set-cookie'])['a_session_' . $id]; + $session = $response['cookies']['a_session_' . $id]; /** * Test for SUCCESS @@ -1315,7 +1315,7 @@ class ProjectsConsoleClientTest extends Scope 'password' => $password, ]); $this->assertEquals(201, $session['headers']['status-code']); - $session = $this->client->parseCookie((string)$session['headers']['set-cookie'])['a_session_' . $id]; + $session = $session['cookies']['a_session_' . $id]; $response = $this->client->call(Client::METHOD_PATCH, '/account/password', array_merge([ 'origin' => 'http://localhost', diff --git a/tests/e2e/Services/Realtime/RealtimeCustomClientTest.php b/tests/e2e/Services/Realtime/RealtimeCustomClientTest.php index 56796280a..e8baa2044 100644 --- a/tests/e2e/Services/Realtime/RealtimeCustomClientTest.php +++ b/tests/e2e/Services/Realtime/RealtimeCustomClientTest.php @@ -468,7 +468,7 @@ class RealtimeCustomClientTest extends Scope 'password' => 'new-password', ]); - $sessionNew = $this->client->parseCookie((string)$response['headers']['set-cookie'])['a_session_' . $projectId]; + $sessionNew = $response['cookies']['a_session_' . $projectId]; $sessionNewId = $response['body']['$id']; return array("session" => $sessionNew, "sessionId" => $sessionNewId); diff --git a/tests/e2e/Services/Storage/StorageBase.php b/tests/e2e/Services/Storage/StorageBase.php index 36d2ee236..c4a15585e 100644 --- a/tests/e2e/Services/Storage/StorageBase.php +++ b/tests/e2e/Services/Storage/StorageBase.php @@ -74,10 +74,7 @@ trait StorageBase 'name' => 'Test Bucket 2', 'fileSecurity' => true, 'permissions' => [ - Permission::read(Role::any()), Permission::create(Role::any()), - Permission::update(Role::any()), - Permission::delete(Role::any()), ], ]); $this->assertEquals(201, $bucket2['headers']['status-code']); @@ -110,9 +107,7 @@ trait StorageBase 'fileId' => $fileId, 'file' => $curlFile, 'permissions' => [ - Permission::read(Role::any()), - Permission::update(Role::any()), - Permission::delete(Role::any()), + Permission::read(Role::any()) ], ]); $counter++; diff --git a/tests/e2e/Services/Storage/StoragePermissionsScope.php b/tests/e2e/Services/Storage/StoragePermissionsScope.php index 09b35fad3..546931601 100644 --- a/tests/e2e/Services/Storage/StoragePermissionsScope.php +++ b/tests/e2e/Services/Storage/StoragePermissionsScope.php @@ -32,7 +32,7 @@ trait StoragePermissionsScope 'password' => $password, ]); - $session = $this->client->parseCookie((string)$session['headers']['set-cookie'])['a_session_' . $this->getProject()['$id']]; + $session = $session['cookies']['a_session_' . $this->getProject()['$id']]; $user = [ diff --git a/tests/e2e/Services/Teams/TeamsBaseClient.php b/tests/e2e/Services/Teams/TeamsBaseClient.php index 7b6b43955..2c2ff02c4 100644 --- a/tests/e2e/Services/Teams/TeamsBaseClient.php +++ b/tests/e2e/Services/Teams/TeamsBaseClient.php @@ -403,7 +403,7 @@ trait TeamsBaseClient $this->assertCount(2, $response['body']['roles']); $this->assertEquals(true, (new DatetimeValidator())->isValid($response['body']['joined'])); $this->assertEquals(true, $response['body']['confirm']); - $session = $this->client->parseCookie((string)$response['headers']['set-cookie'])['a_session_' . $this->getProject()['$id']]; + $session = $response['cookies']['a_session_' . $this->getProject()['$id']]; $data['session'] = $session; $response = $this->client->call(Client::METHOD_GET, '/account', array_merge([ diff --git a/tests/e2e/Services/Webhooks/WebhooksCustomClientTest.php b/tests/e2e/Services/Webhooks/WebhooksCustomClientTest.php index 21c270d19..4ad0a96e5 100644 --- a/tests/e2e/Services/Webhooks/WebhooksCustomClientTest.php +++ b/tests/e2e/Services/Webhooks/WebhooksCustomClientTest.php @@ -107,7 +107,7 @@ class WebhooksCustomClientTest extends Scope $this->assertEquals($accountSession['headers']['status-code'], 201); $id = $account['body']['$id']; - $session = $this->client->parseCookie((string)$accountSession['headers']['set-cookie'])['a_session_' . $this->getProject()['$id']]; + $session = $accountSession['cookies']['a_session_' . $this->getProject()['$id']]; $account = $this->client->call(Client::METHOD_PATCH, '/account/status', array_merge([ 'origin' => 'http://localhost', @@ -170,7 +170,7 @@ class WebhooksCustomClientTest extends Scope $this->assertEquals($accountSession['headers']['status-code'], 201); $sessionId = $accountSession['body']['$id']; - $session = $this->client->parseCookie((string)$accountSession['headers']['set-cookie'])['a_session_' . $this->getProject()['$id']]; + $session = $accountSession['cookies']['a_session_' . $this->getProject()['$id']]; $webhook = $this->getLastRequest(); $signatureKey = $this->getProject()['signatureKey']; @@ -248,7 +248,7 @@ class WebhooksCustomClientTest extends Scope ]); $sessionId = $accountSession['body']['$id']; - $session = $this->client->parseCookie((string)$accountSession['headers']['set-cookie'])['a_session_' . $this->getProject()['$id']]; + $session = $accountSession['cookies']['a_session_' . $this->getProject()['$id']]; $this->assertEquals($accountSession['headers']['status-code'], 201); @@ -334,7 +334,7 @@ class WebhooksCustomClientTest extends Scope ]); $sessionId = $accountSession['body']['$id']; - $session = $this->client->parseCookie((string)$accountSession['headers']['set-cookie'])['a_session_' . $this->getProject()['$id']]; + $session = $accountSession['cookies']['a_session_' . $this->getProject()['$id']]; $this->assertEquals($accountSession['headers']['status-code'], 201); @@ -407,7 +407,7 @@ class WebhooksCustomClientTest extends Scope $this->assertEquals($accountSession['headers']['status-code'], 201); $sessionId = $accountSession['body']['$id']; - $session = $this->client->parseCookie((string)$accountSession['headers']['set-cookie'])['a_session_' . $this->getProject()['$id']]; + $session = $accountSession['cookies']['a_session_' . $this->getProject()['$id']]; return array_merge($data, [ 'sessionId' => $sessionId,