diff --git a/.env b/.env index f89386b5c..09abb07be 100644 --- a/.env +++ b/.env @@ -79,7 +79,7 @@ _APP_MAINTENANCE_RETENTION_CACHE=2592000 _APP_MAINTENANCE_RETENTION_EXECUTION=1209600 _APP_MAINTENANCE_RETENTION_ABUSE=86400 _APP_MAINTENANCE_RETENTION_AUDIT=1209600 -_APP_USAGE_AGGREGATION_INTERVAL=60000 +_APP_USAGE_AGGREGATION_INTERVAL=30 _APP_MAINTENANCE_RETENTION_USAGE_HOURLY=8640000 _APP_MAINTENANCE_RETENTION_SCHEDULES=86400 _APP_USAGE_STATS=enabled diff --git a/Dockerfile b/Dockerfile index 1075f9a12..9d8e26b29 100755 --- a/Dockerfile +++ b/Dockerfile @@ -105,7 +105,8 @@ RUN chmod +x /usr/local/bin/doctor && \ chmod +x /usr/local/bin/worker-migrations && \ chmod +x /usr/local/bin/worker-webhooks && \ chmod +x /usr/local/bin/worker-hamster && \ - chmod +x /usr/local/bin/worker-usage + chmod +x /usr/local/bin/worker-usage && \ + chmod +x /usr/local/bin/worker-usage-dump # Cloud Executabless diff --git a/app/config/collections.php b/app/config/collections.php index cdc5d03b8..1ab9c42fa 100644 --- a/app/config/collections.php +++ b/app/config/collections.php @@ -34,6 +34,28 @@ $commonCollections = [ 'array' => false, 'filters' => [], ], + [ + '$id' => 'resourceType', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 255, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => ID::custom('mimeType'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 255, // https://tools.ietf.org/html/rfc4288#section-4.2 + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], [ '$id' => 'accessedAt', 'type' => Database::VAR_DATETIME, diff --git a/app/config/errors.php b/app/config/errors.php index 9e41d4312..67182c7e6 100644 --- a/app/config/errors.php +++ b/app/config/errors.php @@ -272,6 +272,11 @@ return [ 'description' => 'User phone is already verified', 'code' => 409 ], + Exception::USER_DELETION_PROHIBITED => [ + 'name' => Exception::USER_DELETION_PROHIBITED, + 'description' => 'User deletion is not allowed for users with active memberships. Please delete all confirmed memberships before deleting the account.', + 'code' => 400 + ], Exception::USER_TARGET_NOT_FOUND => [ 'name' => Exception::USER_TARGET_NOT_FOUND, 'description' => 'The target could not be found.', diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index 11facf04a..07b88210c 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -2124,7 +2124,7 @@ App::get('/v1/account/logs') } $response->dynamic(new Document([ - 'total' => $audit->countLogsByUser($user->getId()), + 'total' => $audit->countLogsByUser($user->getInternalId()), 'logs' => $output, ]), Response::MODEL_LOG_LIST); }); @@ -3791,15 +3791,27 @@ App::delete('/v1/account') ->label('sdk.response.code', Response::STATUS_CODE_NOCONTENT) ->label('sdk.response.model', Response::MODEL_NONE) ->inject('user') + ->inject('project') ->inject('response') ->inject('dbForProject') ->inject('queueForEvents') ->inject('queueForDeletes') - ->action(function (Document $user, Response $response, Database $dbForProject, Event $queueForEvents, Delete $queueForDeletes) { + ->action(function (Document $user, Document $project, Response $response, Database $dbForProject, Event $queueForEvents, Delete $queueForDeletes) { if ($user->isEmpty()) { throw new Exception(Exception::USER_NOT_FOUND); } + if ($project->getId() === 'console') { + // get all memberships + $memberships = $user->getAttribute('memberships', []); + foreach ($memberships as $membership) { + // prevent deletion if at least one active membership + if ($membership->getAttribute('confirm', false)) { + throw new Exception(Exception::USER_DELETION_PROHIBITED); + } + } + } + $dbForProject->deleteDocument('users', $user->getId()); $queueForDeletes diff --git a/app/controllers/api/avatars.php b/app/controllers/api/avatars.php index ffcfb8f10..fa4b20637 100644 --- a/app/controllers/api/avatars.php +++ b/app/controllers/api/avatars.php @@ -76,7 +76,7 @@ $getUserGitHub = function (string $userId, Document $project, Database $dbForPro } if (empty($gitHubSession)) { - throw new Exception(Exception::GENERAL_UNKNOWN, 'GitHub session not found.'); + throw new Exception(Exception::USER_SESSION_NOT_FOUND, 'GitHub session not found.'); } $provider = $gitHubSession->getAttribute('provider', ''); diff --git a/app/controllers/api/project.php b/app/controllers/api/project.php index 8dd6f8b82..a067c4558 100644 --- a/app/controllers/api/project.php +++ b/app/controllers/api/project.php @@ -144,9 +144,30 @@ App::get('/v1/project/usage') ]; }, $dbForProject->find('buckets')); + // merge network inbound + outbound + $projectBandwidth = []; + foreach ($usage[METRIC_NETWORK_INBOUND] as $item) { + $projectBandwidth[$item['date']] ??= 0; + $projectBandwidth[$item['date']] += $item['value']; + } + + foreach ($usage[METRIC_NETWORK_OUTBOUND] as $item) { + $projectBandwidth[$item['date']] ??= 0; + $projectBandwidth[$item['date']] += $item['value']; + } + + + $network = []; + foreach ($projectBandwidth as $date => $value) { + $network[] = [ + 'date' => $date, + 'value' => $value + ]; + } + $response->dynamic(new Document([ 'requests' => ($usage[METRIC_NETWORK_REQUESTS]), - 'network' => ($usage[METRIC_NETWORK_INBOUND] + $usage[METRIC_NETWORK_OUTBOUND]), + 'network' => $network, 'users' => ($usage[METRIC_USERS]), 'executions' => ($usage[METRIC_EXECUTIONS]), 'executionsTotal' => $total[METRIC_EXECUTIONS], diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index 4968fc381..afd0d9fa5 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -1482,6 +1482,7 @@ App::delete('/v1/storage/buckets/:bucketId/files/:fileId') if ($deviceDeleted) { $queueForDeletes ->setType(DELETE_TYPE_CACHE_BY_RESOURCE) + ->setResourceType('bucket/' . $bucket->getId()) ->setResource('file/' . $fileId) ; diff --git a/app/controllers/api/teams.php b/app/controllers/api/teams.php index d5db918cd..c1fcb9d8c 100644 --- a/app/controllers/api/teams.php +++ b/app/controllers/api/teams.php @@ -995,7 +995,7 @@ App::patch('/v1/teams/:teamId/memberships/:membershipId/status') $dbForProject->purgeCachedDocument('users', $user->getId()); - $team = Authorization::skip(fn() => $dbForProject->updateDocument('teams', $team->getId(), $team->setAttribute('total', $team->getAttribute('total', 0) + 1))); + Authorization::skip(fn() => $dbForProject->increaseDocumentAttribute('teams', $team->getId(), 'total', 1)); $queueForEvents ->setParam('teamId', $team->getId()) @@ -1075,8 +1075,7 @@ App::delete('/v1/teams/:teamId/memberships/:membershipId') $dbForProject->purgeCachedDocument('users', $user->getId()); if ($membership->getAttribute('confirm')) { // Count only confirmed members - $team->setAttribute('total', \max($team->getAttribute('total', 0) - 1, 0)); - Authorization::skip(fn() => $dbForProject->updateDocument('teams', $team->getId(), $team)); + Authorization::skip(fn() => $dbForProject->decreaseDocumentAttribute('teams', $team->getId(), 'total', 1, 0)); } $queueForEvents diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index 55ad75863..e2c41d9ed 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -192,7 +192,6 @@ App::post('/v1/users') ->inject('hooks') ->action(function (string $userId, ?string $email, ?string $phone, ?string $password, string $name, Response $response, Document $project, Database $dbForProject, Event $queueForEvents, Hooks $hooks) { $user = createUser('plaintext', '{}', $userId, $email, $password, $phone, $name, $project, $dbForProject, $queueForEvents, $hooks); - $response ->setStatusCode(Response::STATUS_CODE_CREATED) ->dynamic($user, Response::MODEL_USER); @@ -806,6 +805,9 @@ App::get('/v1/users/:userId/logs') $output[$i] = new Document([ 'event' => $log['event'], + 'userId' => ID::custom($log['data']['userId']), + 'userEmail' => $log['data']['userEmail'] ?? null, + 'userName' => $log['data']['userName'] ?? null, 'ip' => $log['ip'], 'time' => $log['time'], 'osCode' => $os['osCode'], @@ -834,7 +836,7 @@ App::get('/v1/users/:userId/logs') } $response->dynamic(new Document([ - 'total' => $audit->countLogsByUser($user->getId()), + 'total' => $audit->countLogsByUser($user->getInternalId()), 'logs' => $output, ]), Response::MODEL_LOG_LIST); }); diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 141553e12..b9429fab6 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -403,24 +403,22 @@ App::init() ; $useCache = $route->getLabel('cache', false); - if ($useCache) { $key = md5($request->getURI() . implode('*', $request->getParams()) . '*' . APP_CACHE_BUSTER); + $cacheLog = Authorization::skip(fn () => $dbForProject->getDocument('cache', $key)); $cache = new Cache( new Filesystem(APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $project->getId()) ); $timestamp = 60 * 60 * 24 * 30; $data = $cache->load($key, $timestamp); - if (!empty($data)) { - $data = json_decode($data, true); - $parts = explode('/', $data['resourceType']); + if (!empty($data) && !$cacheLog->isEmpty()) { + $parts = explode('/', $cacheLog->getAttribute('resourceType')); $type = $parts[0] ?? null; if ($type === 'bucket') { $bucketId = $parts[1] ?? null; - - $bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); + $bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); $isAPIKey = Auth::isAppUser(Authorization::getRoles()); $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles()); @@ -432,11 +430,12 @@ App::init() $fileSecurity = $bucket->getAttribute('fileSecurity', false); $validator = new Authorization(Database::PERMISSION_READ); $valid = $validator->isValid($bucket->getRead()); + if (!$fileSecurity && !$valid) { throw new Exception(Exception::USER_UNAUTHORIZED); } - $parts = explode('/', $data['resource']); + $parts = explode('/', $cacheLog->getAttribute('resource')); $fileId = $parts[1] ?? null; if ($fileSecurity && !$valid) { @@ -453,8 +452,8 @@ App::init() $response ->addHeader('Expires', \date('D, d M Y H:i:s', \time() + $timestamp) . ' GMT') ->addHeader('X-Appwrite-Cache', 'hit') - ->setContentType($data['contentType']) - ->send(base64_decode($data['payload'])) + ->setContentType($cacheLog->getAttribute('mimeType')) + ->send($data) ; } else { $response->addHeader('X-Appwrite-Cache', 'miss'); @@ -635,7 +634,6 @@ App::shutdown() if ($useCache) { $resource = $resourceType = null; $data = $response->getPayload(); - if (!empty($data['payload'])) { $pattern = $route->getLabel('cache.resource', null); if (!empty($pattern)) { @@ -647,24 +645,19 @@ App::shutdown() $resourceType = $parseLabel($pattern, $responsePayload, $requestParams, $user); } - $key = md5($request->getURI() . implode('*', $request->getParams()) . '*' . APP_CACHE_BUSTER); - $data = json_encode([ - 'resourceType' => $resourceType, - 'resource' => $resource, - 'contentType' => $response->getContentType(), - 'payload' => base64_encode($data['payload']), - ]); - - $signature = md5($data); - $cacheLog = Authorization::skip(fn () => $dbForProject->getDocument('cache', $key)); + $key = md5($request->getURI() . '*' . implode('*', $request->getParams())) . '*' . APP_CACHE_BUSTER; + $signature = md5($data['payload']); + $cacheLog = Authorization::skip(fn () => $dbForProject->getDocument('cache', $key)); $accessedAt = $cacheLog->getAttribute('accessedAt', ''); $now = DateTime::now(); if ($cacheLog->isEmpty()) { Authorization::skip(fn () => $dbForProject->createDocument('cache', new Document([ - '$id' => $key, - 'resource' => $resource, - 'accessedAt' => $now, - 'signature' => $signature, + '$id' => $key, + 'resource' => $resource, + 'resourceType' => $resourceType, + 'mimeType' => $response->getContentType(), + 'accessedAt' => $now, + 'signature' => $signature, ]))); } elseif (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_CACHE_UPDATE)) > $accessedAt) { $cacheLog->setAttribute('accessedAt', $now); @@ -675,7 +668,7 @@ App::shutdown() $cache = new Cache( new Filesystem(APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $project->getId()) ); - $cache->save($key, $data); + $cache->save($key, $data['payload']); } } } diff --git a/app/worker.php b/app/worker.php index cd0601afa..d213bb07e 100644 --- a/app/worker.php +++ b/app/worker.php @@ -15,6 +15,7 @@ use Appwrite\Event\Messaging; use Appwrite\Event\Migration; use Appwrite\Event\Phone; use Appwrite\Event\Usage; +use Appwrite\Event\UsageDump; use Appwrite\Platform\Appwrite; use Swoole\Runtime; use Utopia\App; @@ -146,6 +147,10 @@ Server::setResource('queueForUsage', function (Connection $queue) { return new Usage($queue); }, ['queue']); +Server::setResource('queueForUsageDump', function (Connection $queue) { + return new UsageDump($queue); +}, ['queue']); + Server::setResource('queue', function (Group $pools) { return $pools->get('queue')->pop()->getResource(); }, ['pools']); @@ -315,12 +320,9 @@ $worker Console::error('[Error] Line: ' . $error->getLine()); }); -try { - $workerStart = $worker->getWorkerStart(); -} catch (\Throwable $error) { - $worker->workerStart(); -} finally { - Console::info("Worker $workerName started"); -} +$worker->workerStart() + ->action(function () use ($workerName) { + Console::info("Worker $workerName started"); + }); $worker->start(); diff --git a/bin/worker-usage-dump b/bin/worker-usage-dump new file mode 100644 index 000000000..43ca87fcb --- /dev/null +++ b/bin/worker-usage-dump @@ -0,0 +1,3 @@ +#!/bin/sh + +php /usr/src/code/app/worker.php usage-dump $@ \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 0a806929d..f5754b4b0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -582,7 +582,6 @@ services: - _APP_REDIS_PORT - _APP_REDIS_USER - _APP_REDIS_PASS - - _APP_SMS_PROJECTS_DENY_LIST - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA @@ -592,6 +591,7 @@ services: - _APP_LOGGING_CONFIG - _APP_SMS_FROM - _APP_SMS_PROVIDER + - _APP_SMS_PROJECTS_DENY_LIST appwrite-worker-migrations: entrypoint: worker-migrations @@ -696,6 +696,37 @@ services: - _APP_LOGGING_CONFIG - _APP_USAGE_AGGREGATION_INTERVAL + appwrite-worker-usage-dump: + entrypoint: worker-usage-dump + <<: *x-logging + container_name: appwrite-worker-usage-dump + image: appwrite-dev + networks: + - appwrite + volumes: + - ./app:/usr/src/code/app + - ./src:/usr/src/code/src + depends_on: + - redis + - mariadb + environment: + - _APP_ENV + - _APP_WORKER_PER_CORE + - _APP_OPENSSL_KEY_V1 + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS + - _APP_USAGE_STATS + - _APP_LOGGING_PROVIDER + - _APP_LOGGING_CONFIG + - _APP_USAGE_AGGREGATION_INTERVAL + appwrite-scheduler-functions: entrypoint: schedule-functions <<: *x-logging @@ -941,6 +972,7 @@ services: # - appwrite # volumes: # - appwrite-uploads:/storage/uploads + # Dev Tools Start ------------------------------------------------------------------------------------------ # # The Appwrite Team uses the following tools to help debug, monitor and diagnose the Appwrite stack @@ -949,8 +981,9 @@ services: # # MailCatcher - An SMTP server. Catches all system emails and displays them in a nice UI. # RequestCatcher - An HTTP server. Catches all system https calls and displays them using a simple HTTP API. Used to debug & tests webhooks and HTTP tasks - # RedisCommander - A nice UI for exploring Redis data - # Webgrind - A nice UI for exploring and debugging code-level stuff + # Redis Insight - A nice UI for exploring Redis data + # Adminer - A nice UI for exploring MariaDB data + # GraphQl Explorer - A nice UI for exploring GraphQL API maildev: # used mainly for dev tests image: appwrite/mailcatcher:1.0.0 @@ -980,21 +1013,15 @@ services: networks: - appwrite - # redis-commander: - # image: rediscommander/redis-commander:latest - # restart: unless-stopped - # networks: - # - appwrite - # environment: - # - REDIS_HOSTS=redis - # ports: - # - "8081:8081" - # webgrind: - # image: 'jokkedk/webgrind:latest' - # volumes: - # - './debug:/tmp' - # ports: - # - '3001:80' + redis-insight: + image: redis/redisinsight:latest + restart: unless-stopped + networks: + - appwrite + environment: + - REDIS_HOSTS=redis + ports: + - "8081:5540" graphql-explorer: container_name: appwrite-graphql-explorer diff --git a/docs/sdks/android/GETTING_STARTED.md b/docs/sdks/android/GETTING_STARTED.md index 684f98e0f..de16f2cc5 100644 --- a/docs/sdks/android/GETTING_STARTED.md +++ b/docs/sdks/android/GETTING_STARTED.md @@ -52,8 +52,9 @@ When trying to connect to Appwrite from an emulator or a mobile device, localhos val account = Account(client) val response = account.create( ID.unique(), - "email@example.com", - "password" + "email@example.com", + "password", + "Walter O'Brien" ) ``` @@ -72,8 +73,9 @@ val client = Client(context) val account = Account(client) val user = account.create( ID.unique(), - "email@example.com", - "password" + "email@example.com", + "password", + "Walter O'Brien" ) ``` @@ -82,7 +84,7 @@ The Appwrite Android SDK raises an `AppwriteException` object with `message`, `c ```kotlin try { - var user = account.create(ID.unique(), "email@example.com", "password") + var user = account.create(ID.unique(),"email@example.com","password","Walter O'Brien") Log.d("Appwrite user", user.toMap()) } catch(e : AppwriteException) { e.printStackTrace() @@ -94,4 +96,4 @@ You can use the following resources to learn more and get help - 🚀 [Getting Started Tutorial](https://appwrite.io/docs/getting-started-for-android) - 📜 [Appwrite Docs](https://appwrite.io/docs) - 💬 [Discord Community](https://appwrite.io/discord) -- 🚂 [Appwrite Android Playground](https://github.com/appwrite/playground-for-android) \ No newline at end of file +- 🚂 [Appwrite Android Playground](https://github.com/appwrite/playground-for-android) diff --git a/docs/sdks/apple/GETTING_STARTED.md b/docs/sdks/apple/GETTING_STARTED.md index e62f1ce3f..6defbc5f0 100644 --- a/docs/sdks/apple/GETTING_STARTED.md +++ b/docs/sdks/apple/GETTING_STARTED.md @@ -75,9 +75,10 @@ let account = Account(client) do { let user = try await account.create( - userId: ID.unique(), - email: "email@example.com", - password: "password" + userId: ID.unique(), + email: "email@example.com", + password: "password", + name: "Walter O'Brien" ) print(String(describing: user.toMap())) } catch { @@ -100,9 +101,10 @@ func main() { do { let user = try await account.create( - userId: ID.unique(), - email: "email@example.com", - password: "password" + userId: ID.unique(), + email: "email@example.com", + password: "password", + name: "Walter O'Brien" ) print(String(describing: account.toMap())) } catch { diff --git a/docs/sdks/dart/EXAMPLES.md b/docs/sdks/dart/EXAMPLES.md index 208cc7f78..fc2c6d099 100644 --- a/docs/sdks/dart/EXAMPLES.md +++ b/docs/sdks/dart/EXAMPLES.md @@ -18,9 +18,11 @@ Create a new user: Users users = Users(client); User result = await users.create( - userId: '[USER_ID]', - email: 'email@example.com', - password: 'password', + userId: ID.unique(), + email: "email@example.com", + phone: "+123456789", + password: "password", + name: "Walter O'Brien" ); ``` @@ -57,4 +59,4 @@ storage.createFile( }); ``` -All examples and API features are available at the [official Appwrite docs](https://appwrite.io/docs) \ No newline at end of file +All examples and API features are available at the [official Appwrite docs](https://appwrite.io/docs) diff --git a/docs/sdks/dart/GETTING_STARTED.md b/docs/sdks/dart/GETTING_STARTED.md index 559c0894a..a1dd4b5c4 100644 --- a/docs/sdks/dart/GETTING_STARTED.md +++ b/docs/sdks/dart/GETTING_STARTED.md @@ -16,7 +16,7 @@ void main() async { Users users = Users(client); try { - final user = await users.create(userId: ID.unique(), email: ‘email@example.com’,password: ‘password’, name: ‘name’); + final user = await users.create(userId: ID.unique(), email: "email@example.com", phone: "+123456789", password: "password", name: "Walter O'Brien"); print(user.toMap()); } on AppwriteException catch(e) { print(e.message); @@ -31,7 +31,7 @@ The Appwrite Dart SDK raises `AppwriteException` object with `message`, `code` a Users users = Users(client); try { - final user = await users.create(userId: ID.unique(), email: ‘email@example.com’,password: ‘password’, name: ‘name’); + final user = await users.create(userId: ID.unique(), email: "email@example.com", phone: "+123456789", password: "password", name: "Walter O'Brien"); print(user.toMap()); } on AppwriteException catch(e) { //show message to user or do other operation based on error as required diff --git a/docs/sdks/deno/GETTING_STARTED.md b/docs/sdks/deno/GETTING_STARTED.md index 6546719a6..22ea80aa8 100644 --- a/docs/sdks/deno/GETTING_STARTED.md +++ b/docs/sdks/deno/GETTING_STARTED.md @@ -21,7 +21,7 @@ Once your SDK object is set, create any of the Appwrite service objects and choo ```typescript let users = new sdk.Users(client); -let user = await users.create(ID.unique(), 'email@example.com', 'password'); +let user = await users.create(ID.unique(), "email@example.com", "+123456789", "password", "Walter O'Brien"); console.log(user); ``` @@ -39,7 +39,7 @@ client .setSelfSigned() // Use only on dev mode with a self-signed SSL cert ; -let user = await users.create(ID.unique(), 'email@example.com', 'password'); +let user = await users.create(ID.unique(), "email@example.com", "+123456789", "password", "Walter O'Brien"); console.log(user); ``` @@ -50,7 +50,7 @@ The Appwrite Deno SDK raises `AppwriteException` object with `message`, `code` a let users = new sdk.Users(client); try { - let user = await users.create(ID.unique(), 'email@example.com', 'password'); + let user = await users.create(ID.unique(), "email@example.com", "+123456789", "password", "Walter O'Brien"); } catch(e) { console.log(e.message); } diff --git a/docs/sdks/dotnet/GETTING_STARTED.md b/docs/sdks/dotnet/GETTING_STARTED.md index 08d7742dd..ae1f692e0 100644 --- a/docs/sdks/dotnet/GETTING_STARTED.md +++ b/docs/sdks/dotnet/GETTING_STARTED.md @@ -18,8 +18,9 @@ var users = new Users(client); var user = await users.Create( userId: ID.Unique(), email: "email@example.com", + phone: "+123456789", password: "password", - name: "name"); + name: "Walter O'Brien"); Console.WriteLine(user.ToMap()); ``` @@ -35,8 +36,9 @@ try var user = await users.Create( userId: ID.Unique(), email: "email@example.com", + phone: "+123456789", password: "password", - name: "name"); + name: "Walter O'Brien"); } catch (AppwriteException e) { diff --git a/docs/sdks/flutter-dev/EXAMPLES.md b/docs/sdks/flutter-dev/EXAMPLES.md index 23b631900..d0cb0c2b2 100644 --- a/docs/sdks/flutter-dev/EXAMPLES.md +++ b/docs/sdks/flutter-dev/EXAMPLES.md @@ -17,7 +17,7 @@ Create a new user and session: ```dart Account account = Account(client); -final user = await account.create(userId: '[USER_ID]', email: 'me@appwrite.io', password: 'password', name: 'My Name'); +final user = await account.create(userId: ID.unique(), email: "email@example.com", password: "password", name: "Walter O'Brien"); final session = await account.createEmailSession(email: 'me@appwrite.io', password: 'password'); @@ -60,4 +60,4 @@ storage.createFile( }); ``` -All examples and API features are available at the [official Appwrite docs](https://appwrite.io/docs) \ No newline at end of file +All examples and API features are available at the [official Appwrite docs](https://appwrite.io/docs) diff --git a/docs/sdks/flutter/EXAMPLES.md b/docs/sdks/flutter/EXAMPLES.md index 23b631900..d0cb0c2b2 100644 --- a/docs/sdks/flutter/EXAMPLES.md +++ b/docs/sdks/flutter/EXAMPLES.md @@ -17,7 +17,7 @@ Create a new user and session: ```dart Account account = Account(client); -final user = await account.create(userId: '[USER_ID]', email: 'me@appwrite.io', password: 'password', name: 'My Name'); +final user = await account.create(userId: ID.unique(), email: "email@example.com", password: "password", name: "Walter O'Brien"); final session = await account.createEmailSession(email: 'me@appwrite.io', password: 'password'); @@ -60,4 +60,4 @@ storage.createFile( }); ``` -All examples and API features are available at the [official Appwrite docs](https://appwrite.io/docs) \ No newline at end of file +All examples and API features are available at the [official Appwrite docs](https://appwrite.io/docs) diff --git a/docs/sdks/flutter/GETTING_STARTED.md b/docs/sdks/flutter/GETTING_STARTED.md index 110ee3eb4..8d239402b 100644 --- a/docs/sdks/flutter/GETTING_STARTED.md +++ b/docs/sdks/flutter/GETTING_STARTED.md @@ -105,10 +105,7 @@ When trying to connect to Appwrite from an emulator or a mobile device, localhos Account account = Account(client); final user = await account .create( - userId: ID.unique(), - email: 'me@appwrite.io', - password: 'password', - name: 'My Name' + userId: ID.unique(), email: "email@example.com", password: "password", name: "Walter O'Brien" ); ``` @@ -133,10 +130,7 @@ void main() { final user = await account .create( - userId: ID.unique(), - email: 'me@appwrite.io', - password: 'password', - name: 'My Name' + userId: ID.unique(), email: "email@example.com", password: "password", name: "Walter O'Brien" ); } ``` @@ -148,7 +142,7 @@ The Appwrite Flutter SDK raises `AppwriteException` object with `message`, `type Account account = Account(client); try { - final user = await account.create(userId: ID.unique(), email: ‘email@example.com’,password: ‘password’, name: ‘name’); + final user = await account.create(userId: ID.unique(), email: "email@example.com", password: "password", name: "Walter O'Brien"); print(user.toMap()); } on AppwriteException catch(e) { //show message to user or do other operation based on error as required diff --git a/docs/sdks/kotlin/GETTING_STARTED.md b/docs/sdks/kotlin/GETTING_STARTED.md index 99d5e719a..5b5ee5f05 100644 --- a/docs/sdks/kotlin/GETTING_STARTED.md +++ b/docs/sdks/kotlin/GETTING_STARTED.md @@ -26,7 +26,9 @@ val users = Users(client) val user = users.create( user = ID.unique(), email = "email@example.com", + phone = "+123456789", password = "password", + name = "Walter O'Brien" ) ``` @@ -48,7 +50,9 @@ suspend fun main() { val user = users.create( user = ID.unique(), email = "email@example.com", + phone = "+123456789", password = "password", + name = "Walter O'Brien" ) } ``` @@ -68,7 +72,9 @@ suspend fun main() { val user = users.create( user = ID.unique(), email = "email@example.com", + phone = "+123456789", password = "password", + name = "Walter O'Brien" ) } catch (e: AppwriteException) { e.printStackTrace() diff --git a/docs/sdks/nodejs/GETTING_STARTED.md b/docs/sdks/nodejs/GETTING_STARTED.md index 985648d3f..e98400f84 100644 --- a/docs/sdks/nodejs/GETTING_STARTED.md +++ b/docs/sdks/nodejs/GETTING_STARTED.md @@ -22,7 +22,7 @@ Once your SDK object is set, create any of the Appwrite service objects and choo ```js let users = new sdk.Users(client); -let promise = users.create(sdk.ID.unique(), 'email@example.com', undefined, 'password', 'Jane Doe'); +let promise = users.create(sdk.ID.unique(), "email@example.com", "+123456789", "password", "Walter O'Brien"); promise.then(function (response) { console.log(response); @@ -45,7 +45,7 @@ client ; let users = new sdk.Users(client); -let promise = users.create(sdk.ID.unique(), 'email@example.com', undefined, 'password', 'Jane Doe'); +let promise = users.create(sdk.ID.unique(), "email@example.com", "+123456789", "password", "Walter O'Brien"); promise.then(function (response) { console.log(response); @@ -61,7 +61,7 @@ The Appwrite Node SDK raises `AppwriteException` object with `message`, `code` a let users = new sdk.Users(client); try { - let res = await users.create(sdk.ID.unique(), 'email@example.com', 'password'); + let res = await users.create(sdk.ID.unique(), "email@example.com", "+123456789", "password", "Walter O'Brien"); } catch(e) { console.log(e.message); } diff --git a/docs/sdks/php/GETTING_STARTED.md b/docs/sdks/php/GETTING_STARTED.md index faa3dcf65..acbd06c8a 100644 --- a/docs/sdks/php/GETTING_STARTED.md +++ b/docs/sdks/php/GETTING_STARTED.md @@ -20,7 +20,7 @@ Once your SDK object is set, create any of the Appwrite service objects and choo ```php $users = new Users($client); -$user = $users->create(ID::unique(), 'email@example.com', 'password'); +$user = $users->create(ID::unique(), "email@example.com", "+123456789", "password", "Walter O'Brien"); ``` ### Full Example @@ -40,7 +40,7 @@ $client $users = new Users($client); -$user = $users->create(ID::unique(), 'email@example.com', 'password'); +$user = $users->create(ID::unique(), "email@example.com", "+123456789", "password", "Walter O'Brien"); ``` ### Error Handling @@ -49,7 +49,7 @@ The Appwrite PHP SDK raises `AppwriteException` object with `message`, `code` an ```php $users = new Users($client); try { - $user = $users->create(ID::unique(), 'email@example.com', 'password'); + $user = $users->create(ID::unique(), "email@example.com", "+123456789", "password", "Walter O'Brien"); } catch(AppwriteException $error) { echo $error->message; } diff --git a/docs/sdks/python/GETTING_STARTED.md b/docs/sdks/python/GETTING_STARTED.md index 9f693b65c..2732ef848 100644 --- a/docs/sdks/python/GETTING_STARTED.md +++ b/docs/sdks/python/GETTING_STARTED.md @@ -23,7 +23,7 @@ Once your SDK object is set, create any of the Appwrite service objects and choo ```python users = Users(client) -result = users.create('[USER_ID]', 'email@example.com', 'password') +result = users.create(ID.unique(), email = "email@example.com", phone = "+123456789", password = "password", name = "Walter O'Brien") ``` ### Full Example @@ -43,7 +43,7 @@ client = Client() users = Users(client) -result = users.create(ID.unique(), 'email@example.com', 'password') +result = users.create(ID.unique(), email = "email@example.com", phone = "+123456789", password = "password", name = "Walter O'Brien") ``` ### Error Handling @@ -52,7 +52,7 @@ The Appwrite Python SDK raises `AppwriteException` object with `message`, `code` ```python users = Users(client) try: - result = users.create(ID.unique(), 'email@example.com', 'password') + result = users.create(ID.unique(), email = "email@example.com", phone = "+123456789", password = "password", name = "Walter O'Brien") except AppwriteException as e: print(e.message) ``` diff --git a/docs/sdks/ruby/GETTING_STARTED.md b/docs/sdks/ruby/GETTING_STARTED.md index da10e1aeb..5d7d7ee37 100644 --- a/docs/sdks/ruby/GETTING_STARTED.md +++ b/docs/sdks/ruby/GETTING_STARTED.md @@ -22,7 +22,7 @@ Once your SDK object is set, create any of the Appwrite service objects and choo ```ruby users = Appwrite::Users.new(client); -user = users.create(userId: Appwrite::ID::unique(), email: 'email@example.com', password: 'password'); +user = users.create(userId: Appwrite::ID::unique(), email: "email@example.com", phone: "+123456789", password: "password", name: "Walter O'Brien"); ``` ### Full Example @@ -40,7 +40,7 @@ client users = Appwrite::Users.new(client); -user = users.create(userId: Appwrite::ID::unique(), email: 'email@example.com', password: 'password'); +user = users.create(userId: Appwrite::ID::unique(), email: "email@example.com", phone: "+123456789", password: "password", name: "Walter O'Brien"); ``` ### Error Handling @@ -50,7 +50,7 @@ The Appwrite Ruby SDK raises `Appwrite::Exception` object with `message`, `code` users = Appwrite::Users.new(client); begin - user = users.create(userId: Appwrite::ID::unique(), email: 'email@example.com', password: 'password'); + user = users.create(userId: Appwrite::ID::unique(), email: "email@example.com", phone: "+123456789", password: "password", name: "Walter O'Brien"); rescue Appwrite::Exception => error puts error.message end diff --git a/docs/sdks/swift/GETTING_STARTED.md b/docs/sdks/swift/GETTING_STARTED.md index e0fb45dd7..49aa51e9b 100644 --- a/docs/sdks/swift/GETTING_STARTED.md +++ b/docs/sdks/swift/GETTING_STARTED.md @@ -25,9 +25,11 @@ let users = Users(client) do { let user = try await users.create( - userId: ID.unique(), - email: "email@example.com", - password: "password" + userId: ID.unique(), + email: "email@example.com", + phone: "+123456789", + password: "password", + name: "Walter O'Brien" ) print(String(describing: user.toMap())) } catch { @@ -51,9 +53,11 @@ func main() { do { let user = try await users.create( - userId: ID.unique(), - email: "email@example.com", - password: "password" + userId: ID.unique(), + email: "email@example.com", + phone: "+123456789", + password: "password", + name: "Walter O'Brien" ) print(String(describing: user.toMap())) } catch { diff --git a/docs/sdks/web/GETTING_STARTED.md b/docs/sdks/web/GETTING_STARTED.md index 445a362c0..26aa9470b 100644 --- a/docs/sdks/web/GETTING_STARTED.md +++ b/docs/sdks/web/GETTING_STARTED.md @@ -25,7 +25,7 @@ Once your SDK object is set, access any of the Appwrite services and choose any const account = new Account(client); // Register User -account.create(ID.unique(), 'me@example.com', 'password', 'Jane Doe') +account.create(ID.unique(), "email@example.com", "password", "Walter O'Brien") .then(function (response) { console.log(response); }, function (error) { @@ -47,7 +47,7 @@ client const account = new Account(client); // Register User -account.create(ID.unique(), 'me@example.com', 'password', 'Jane Doe') +account.create(ID.unique(), "email@example.com", "password", "Walter O'Brien") .then(function (response) { console.log(response); }, function (error) { diff --git a/src/Appwrite/Event/Delete.php b/src/Appwrite/Event/Delete.php index 57300feb7..064fbcefa 100644 --- a/src/Appwrite/Event/Delete.php +++ b/src/Appwrite/Event/Delete.php @@ -10,6 +10,7 @@ class Delete extends Event { protected string $type = ''; protected ?Document $document = null; + protected ?string $resourceType = null; protected ?string $resource = null; protected ?string $datetime = null; protected ?string $hourlyUsageRetentionDatetime = null; @@ -107,6 +108,19 @@ class Delete extends Event return $this; } + /** + * Sets the resource type for the delete event. + * + * @param string $resourceType + * @return self + */ + public function setResourceType(string $resourceType): self + { + $this->resourceType = $resourceType; + + return $this; + } + /** * Returns the set document for the delete event. * @@ -133,6 +147,7 @@ class Delete extends Event 'type' => $this->type, 'document' => $this->document, 'resource' => $this->resource, + 'resourceType' => $this->resourceType, 'datetime' => $this->datetime, 'hourlyUsageRetentionDatetime' => $this->hourlyUsageRetentionDatetime ]); diff --git a/src/Appwrite/Event/Event.php b/src/Appwrite/Event/Event.php index fc12c5b5b..9f71ef5eb 100644 --- a/src/Appwrite/Event/Event.php +++ b/src/Appwrite/Event/Event.php @@ -27,6 +27,9 @@ class Event public const USAGE_QUEUE_NAME = 'v1-usage'; public const USAGE_CLASS_NAME = 'UsageV1'; + public const USAGE_DUMP_QUEUE_NAME = 'v1-usage-dump'; + public const USAGE_DUMP_CLASS_NAME = 'UsageDumpV1'; + public const WEBHOOK_QUEUE_NAME = 'v1-webhooks'; public const WEBHOOK_CLASS_NAME = 'WebhooksV1'; diff --git a/src/Appwrite/Event/Usage.php b/src/Appwrite/Event/Usage.php index 398c3319f..ded276e16 100644 --- a/src/Appwrite/Event/Usage.php +++ b/src/Appwrite/Event/Usage.php @@ -2,6 +2,7 @@ namespace Appwrite\Event; +use Utopia\CLI\Console; use Utopia\Queue\Client; use Utopia\Queue\Connection; use Utopia\Database\Document; diff --git a/src/Appwrite/Event/UsageDump.php b/src/Appwrite/Event/UsageDump.php new file mode 100644 index 000000000..8f8790884 --- /dev/null +++ b/src/Appwrite/Event/UsageDump.php @@ -0,0 +1,47 @@ +setQueue(Event::USAGE_DUMP_QUEUE_NAME) + ->setClass(Event::USAGE_DUMP_CLASS_NAME); + } + + /** + * Add Stats. + * + * @param array $stats + * @return self + */ + public function setStats(array $stats): self + { + $this->stats = $stats; + + return $this; + } + + /** + * Sends metrics to the usage worker. + * + * @return string|bool + */ + public function trigger(): string|bool + { + $client = new Client($this->queue, $this->connection); + + return $client->enqueue([ + 'stats' => $this->stats, + ]); + } +} diff --git a/src/Appwrite/Extend/Exception.php b/src/Appwrite/Extend/Exception.php index b07e31fc3..c8b6cdad0 100644 --- a/src/Appwrite/Extend/Exception.php +++ b/src/Appwrite/Extend/Exception.php @@ -94,6 +94,7 @@ class Exception extends \Exception public const USER_OAUTH2_PROVIDER_ERROR = 'user_oauth2_provider_error'; public const USER_EMAIL_ALREADY_VERIFIED = 'user_email_already_verified'; public const USER_PHONE_ALREADY_VERIFIED = 'user_phone_already_verified'; + public const USER_DELETION_PROHIBITED = 'user_deletion_prohibited'; public const USER_TARGET_NOT_FOUND = 'user_target_not_found'; public const USER_TARGET_ALREADY_EXISTS = 'user_target_already_exists'; diff --git a/src/Appwrite/Platform/Services/Tasks.php b/src/Appwrite/Platform/Services/Tasks.php index 671120a81..e50c7c1dd 100644 --- a/src/Appwrite/Platform/Services/Tasks.php +++ b/src/Appwrite/Platform/Services/Tasks.php @@ -42,7 +42,6 @@ class Tasks extends Service ->addAction(Install::getName(), new Install()) ->addAction(Maintenance::getName(), new Maintenance()) ->addAction(Migrate::getName(), new Migrate()) - ->addAction(Migrate::getName(), new Migrate()) ->addAction(PatchRecreateRepositoriesDocuments::getName(), new PatchRecreateRepositoriesDocuments()) ->addAction(QueueCount::getName(), new QueueCount()) ->addAction(QueueRetry::getName(), new QueueRetry()) diff --git a/src/Appwrite/Platform/Services/Workers.php b/src/Appwrite/Platform/Services/Workers.php index ce0904bcd..84700e82e 100644 --- a/src/Appwrite/Platform/Services/Workers.php +++ b/src/Appwrite/Platform/Services/Workers.php @@ -14,7 +14,7 @@ use Appwrite\Platform\Workers\Builds; use Appwrite\Platform\Workers\Deletes; use Appwrite\Platform\Workers\Hamster; use Appwrite\Platform\Workers\Usage; -use Appwrite\Platform\Workers\UsageHook; +use Appwrite\Platform\Workers\UsageDump; use Appwrite\Platform\Workers\Migrations; class Workers extends Service @@ -33,7 +33,7 @@ class Workers extends Service ->addAction(Messaging::getName(), new Messaging()) ->addAction(Webhooks::getName(), new Webhooks()) ->addAction(Hamster::getName(), new Hamster()) - ->addAction(UsageHook::getName(), new UsageHook()) + ->addAction(UsageDump::getName(), new UsageDump()) ->addAction(Usage::getName(), new Usage()) ->addAction(Migrations::getName(), new Migrations()) diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index d5e5b40d1..40e0de86c 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -71,6 +71,7 @@ class Deletes extends Action $datetime = $payload['datetime'] ?? null; $hourlyUsageRetentionDatetime = $payload['hourlyUsageRetentionDatetime'] ?? null; $resource = $payload['resource'] ?? null; + $resourceType = $payload['resourceType'] ?? null; $document = new Document($payload['document'] ?? []); $project = new Document($payload['project'] ?? []); @@ -80,12 +81,6 @@ class Deletes extends Action switch (\strval($type)) { case DELETE_TYPE_DOCUMENT: switch ($document->getCollection()) { - case DELETE_TYPE_DATABASES: - $this->deleteDatabase($getProjectDB, $document, $project); - break; - case DELETE_TYPE_COLLECTIONS: - $this->deleteCollection($getProjectDB, $document, $project); - break; case DELETE_TYPE_PROJECTS: $this->deleteProject($dbForConsole, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $document); break; @@ -114,10 +109,6 @@ class Deletes extends Action $this->deleteRule($dbForConsole, $document); break; default: - if (\str_starts_with($document->getCollection(), 'database_')) { - $this->deleteCollection($getProjectDB, $document, $project); - break; - } Console::error('No lazy delete operation available for document of type: ' . $document->getCollection()); break; } @@ -147,7 +138,7 @@ class Deletes extends Action $this->deleteUsageStats($project, $getProjectDB, $hourlyUsageRetentionDatetime); break; case DELETE_TYPE_CACHE_BY_RESOURCE: - $this->deleteCacheByResource($project, $getProjectDB, $resource); + $this->deleteCacheByResource($project, $getProjectDB, $resource, $resourceType); break; case DELETE_TYPE_CACHE_BY_TIMESTAMP: $this->deleteCacheByDate($project, $getProjectDB, $datetime); @@ -332,32 +323,37 @@ class Deletes extends Action * @param string $resource * @return void * @throws Authorization + * @param string|null $resourceType + * @throws Exception */ - private function deleteCacheByResource(Document $project, callable $getProjectDB, string $resource): void + private function deleteCacheByResource(Document $project, callable $getProjectDB, string $resource, string $resourceType = null): void { $projectId = $project->getId(); $dbForProject = $getProjectDB($project); - $document = $dbForProject->findOne('cache', [Query::equal('resource', [$resource])]); - if ($document) { - $cache = new Cache( - new Filesystem(APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $projectId) - ); + $cache = new Cache( + new Filesystem(APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $projectId) + ); - $this->deleteById( - $document, - $dbForProject, - function ($document) use ($cache, $projectId) { - $path = APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $projectId . DIRECTORY_SEPARATOR . $document->getId(); - - if ($cache->purge($document->getId())) { - Console::success('Deleting cache file: ' . $path); - } else { - Console::error('Failed to delete cache file: ' . $path); - } - } - ); + $query[] = Query::equal('resource', [$resource]); + if (!empty($resourceType)) { + $query[] = Query::equal('resourceType', [$resourceType]); } + + $this->deleteByGroup( + 'cache', + $query, + $dbForProject, + function (Document $document) use ($cache, $projectId) { + $path = APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $projectId . DIRECTORY_SEPARATOR . $document->getId(); + + if ($cache->purge($document->getId())) { + Console::success('Deleting cache file: ' . $path); + } else { + Console::error('Failed to delete cache file: ' . $path); + } + } + ); } /** @@ -397,72 +393,6 @@ class Deletes extends Action ); } - /** - * @param callable $getProjectDB - * @param Document $document - * @param Document $project - * @return void - * @throws Exception - */ - private function deleteDatabase(callable $getProjectDB, Document $document, Document $project): void - { - $databaseId = $document->getId(); - $dbForProject = $getProjectDB($project); - - $this->deleteByGroup('database_' . $document->getInternalId(), [], $dbForProject, function ($document) use ($getProjectDB, $project) { - $this->deleteCollection($getProjectDB, $document, $project); - }); - - $dbForProject->deleteCollection('database_' . $document->getInternalId()); - $this->deleteAuditLogsByResource($getProjectDB, 'database/' . $databaseId, $project); - } - - /** - * @param callable $getProjectDB - * @param Document $document teams document - * @param Document $project - * @return void - * @throws Exception - */ - private function deleteCollection(callable $getProjectDB, Document $document, Document $project): void - { - $collectionId = $document->getId(); - $collectionInternalId = $document->getInternalId(); - $databaseId = $document->getAttribute('databaseId'); - $databaseInternalId = $document->getAttribute('databaseInternalId'); - - $dbForProject = $getProjectDB($project); - - $relationships = \array_filter( - $document->getAttribute('attributes'), - fn ($attribute) => $attribute['type'] === Database::VAR_RELATIONSHIP - ); - - foreach ($relationships as $relationship) { - if (!$relationship['twoWay']) { - continue; - } - $relatedCollection = $dbForProject->getDocument('database_' . $databaseInternalId, $relationship['relatedCollection']); - $dbForProject->deleteDocument('attributes', $databaseInternalId . '_' . $relatedCollection->getInternalId() . '_' . $relationship['twoWayKey']); - $dbForProject->purgeCachedDocument('database_' . $databaseInternalId, $relatedCollection->getId()); - $dbForProject->purgeCachedCollection('database_' . $databaseInternalId . '_collection_' . $relatedCollection->getInternalId()); - } - - $dbForProject->deleteCollection('database_' . $databaseInternalId . '_collection_' . $document->getInternalId()); - - $this->deleteByGroup('attributes', [ - Query::equal('databaseInternalId', [$databaseInternalId]), - Query::equal('collectionInternalId', [$collectionInternalId]) - ], $dbForProject); - - $this->deleteByGroup('indexes', [ - Query::equal('databaseInternalId', [$databaseInternalId]), - Query::equal('collectionInternalId', [$collectionInternalId]) - ], $dbForProject); - - $this->deleteAuditLogsByResource($getProjectDB, 'database/' . $databaseId . '/collection/' . $collectionId, $project); - } - /** * @param Database $dbForConsole * @param callable $getProjectDB diff --git a/src/Appwrite/Platform/Workers/Messaging.php b/src/Appwrite/Platform/Workers/Messaging.php index 77b46f8c1..62e58db11 100644 --- a/src/Appwrite/Platform/Workers/Messaging.php +++ b/src/Appwrite/Platform/Workers/Messaging.php @@ -69,7 +69,8 @@ class Messaging extends Action * @param Message $message * @param Log $log * @param Database $dbForProject - * @param callable $getLocalCache + * @param Device $deviceForFiles + * @param Device $deviceForLocalFiles * @param Usage $queueForUsage * @return void * @throws \Exception diff --git a/src/Appwrite/Platform/Workers/Usage.php b/src/Appwrite/Platform/Workers/Usage.php index 2271dd590..e2f325250 100644 --- a/src/Appwrite/Platform/Workers/Usage.php +++ b/src/Appwrite/Platform/Workers/Usage.php @@ -3,8 +3,9 @@ namespace Appwrite\Platform\Workers; use Exception; +use Appwrite\Event\UsageDump; +use Utopia\App; use Utopia\CLI\Console; -use Utopia\Database\Database; use Utopia\Database\DateTime; use Utopia\Database\Document; use Utopia\Platform\Action; @@ -12,14 +13,12 @@ use Utopia\Queue\Message; class Usage extends Action { - protected static array $stats = []; - protected array $periods = [ - '1h' => 'Y-m-d H:00', - '1d' => 'Y-m-d 00:00', - 'inf' => '0000-00-00 00:00' - ]; + private array $stats = []; + private int $lastTriggeredTime = 0; + private int $keys = 0; + private const INFINITY_PERIOD = '_inf_'; + private const KEYS_THRESHOLD = 10000; - protected const INFINITY_PERIOD = '_inf_'; public static function getName(): string { return 'usage'; @@ -35,26 +34,31 @@ class Usage extends Action ->desc('Usage worker') ->inject('message') ->inject('getProjectDB') - ->callback(function (Message $message, callable $getProjectDB) { - $this->action($message, $getProjectDB); + ->inject('queueForUsageDump') + ->callback(function (Message $message, callable $getProjectDB, UsageDump $queueForUsageDump) { + $this->action($message, $getProjectDB, $queueForUsageDump); }); + + $this->lastTriggeredTime = time(); } /** * @param Message $message * @param callable $getProjectDB + * @param UsageDump $queueForUsageDump * @return void * @throws \Utopia\Database\Exception * @throws Exception */ - public function action(Message $message, callable $getProjectDB): void + public function action(Message $message, callable $getProjectDB, UsageDump $queueForUsageDump): void { $payload = $message->getPayload() ?? []; if (empty($payload)) { throw new Exception('Missing payload'); } + //Todo Figure out way to preserve keys when the container is being recreated @shimonewman - $payload = $message->getPayload() ?? []; + $aggregationInterval = (int) App::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '20'); $project = new Document($payload['project'] ?? []); $projectId = $project->getInternalId(); foreach ($payload['reduce'] ?? [] as $document) { @@ -70,13 +74,31 @@ class Usage extends Action ); } - self::$stats[$projectId]['project'] = $project; + $this->stats[$projectId]['project'] = $project; foreach ($payload['metrics'] ?? [] as $metric) { - if (!isset(self::$stats[$projectId]['keys'][$metric['key']])) { - self::$stats[$projectId]['keys'][$metric['key']] = $metric['value']; + $this->keys++; + if (!isset($this->stats[$projectId]['keys'][$metric['key']])) { + $this->stats[$projectId]['keys'][$metric['key']] = $metric['value']; continue; } - self::$stats[$projectId]['keys'][$metric['key']] += $metric['value']; + + $this->stats[$projectId]['keys'][$metric['key']] += $metric['value']; + } + + // If keys crossed threshold or X time passed since the last send and there are some keys in the array ($this->stats) + if ( + $this->keys >= self::KEYS_THRESHOLD || + (time() - $this->lastTriggeredTime > $aggregationInterval && $this->keys > 0) + ) { + Console::warning('[' . DateTime::now() . '] Aggregated ' . $this->keys . ' keys'); + + $queueForUsageDump + ->setStats($this->stats) + ->trigger(); + + $this->stats = []; + $this->keys = 0; + $this->lastTriggeredTime = time(); } } @@ -91,7 +113,6 @@ class Usage extends Action */ private function reduce(Document $project, Document $document, array &$metrics, callable $getProjectDB): void { - $dbForProject = $getProjectDB($project); try { diff --git a/src/Appwrite/Platform/Workers/UsageDump.php b/src/Appwrite/Platform/Workers/UsageDump.php new file mode 100644 index 000000000..bc9deda75 --- /dev/null +++ b/src/Appwrite/Platform/Workers/UsageDump.php @@ -0,0 +1,110 @@ + 'Y-m-d H:00', + '1d' => 'Y-m-d 00:00', + 'inf' => '0000-00-00 00:00' + ]; + + public static function getName(): string + { + return 'usage-dump'; + } + + /** + * @throws \Exception + */ + public function __construct() + { + $this + ->inject('message') + ->inject('getProjectDB') + ->callback(function (Message $message, callable $getProjectDB) { + $this->action($message, $getProjectDB); + }); + } + + /** + * @param Message $message + * @param callable $getProjectDB + * @return void + * @throws Exception + * @throws \Utopia\Database\Exception + */ + public function action(Message $message, callable $getProjectDB): void + { + $payload = $message->getPayload() ?? []; + if (empty($payload)) { + throw new Exception('Missing payload'); + } + + // TODO: rename both usage workers @shimonewman + foreach ($payload['stats'] ?? [] as $stats) { + $project = new Document($stats['project'] ?? []); + $numberOfKeys = !empty($stats['keys']) ? count($stats['keys']) : 0; + + if ($numberOfKeys === 0) { + continue; + } + + console::log('[' . DateTime::now() . '] ProjectId [' . $project->getInternalId() . '] Database [' . $project['database'] . '] ' . $numberOfKeys . ' keys'); + + try { + $dbForProject = $getProjectDB($project); + foreach ($stats['keys'] ?? [] as $key => $value) { + if ($value == 0) { + continue; + } + + foreach ($this->periods as $period => $format) { + $time = 'inf' === $period ? null : date($format, time()); + $id = \md5("{$time}_{$period}_{$key}"); + + try { + $dbForProject->createDocument('stats', new Document([ + '$id' => $id, + 'period' => $period, + 'time' => $time, + 'metric' => $key, + 'value' => $value, + 'region' => App::getEnv('_APP_REGION', 'default'), + ])); + } catch (Duplicate $th) { + if ($value < 0) { + $dbForProject->decreaseDocumentAttribute( + 'stats', + $id, + 'value', + abs($value) + ); + } else { + $dbForProject->increaseDocumentAttribute( + 'stats', + $id, + 'value', + $value + ); + } + } + } + } + } catch (\Exception $e) { + console::error('[' . DateTime::now() . '] project [' . $project->getInternalId() . '] database [' . $project['database'] . '] ' . ' ' . $e->getMessage()); + } + } + } +} diff --git a/src/Appwrite/Platform/Workers/UsageHook.php b/src/Appwrite/Platform/Workers/UsageHook.php deleted file mode 100644 index 81729812e..000000000 --- a/src/Appwrite/Platform/Workers/UsageHook.php +++ /dev/null @@ -1,103 +0,0 @@ -setType(Action::TYPE_WORKER_START) - ->inject('register') - ->inject('getProjectDB') - ->callback(function ($register, callable $getProjectDB) { - $this->action($register, $getProjectDB); - }) - ; - } - - /** - * @param $register - * @param $getProjectDB - * @return void - */ - public function action($register, $getProjectDB): void - { - - $interval = (int) App::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '60000'); - Timer::tick($interval, function () use ($register, $getProjectDB) { - - $offset = count(self::$stats); - $projects = array_slice(self::$stats, 0, $offset, true); - array_splice(self::$stats, 0, $offset); - foreach ($projects as $data) { - $numberOfKeys = !empty($data['keys']) ? count($data['keys']) : 0; - $projectInternalId = $data['project']->getInternalId(); - $database = $data['project']['database'] ?? ''; - - console::warning('Ticker started ' . DateTime::now()); - - if ($numberOfKeys === 0) { - continue; - } - - try { - $dbForProject = $getProjectDB($data['project']); - foreach ($data['keys'] ?? [] as $key => $value) { - if ($value == 0) { - continue; - } - - foreach ($this->periods as $period => $format) { - $time = 'inf' === $period ? null : date($format, time()); - $id = \md5("{$time}_{$period}_{$key}"); - - try { - $dbForProject->createDocument('stats', new Document([ - '$id' => $id, - 'period' => $period, - 'time' => $time, - 'metric' => $key, - 'value' => $value, - 'region' => App::getEnv('_APP_REGION', 'default'), - ])); - } catch (Duplicate $th) { - if ($value < 0) { - $dbForProject->decreaseDocumentAttribute( - 'stats', - $id, - 'value', - abs($value) - ); - } else { - $dbForProject->increaseDocumentAttribute( - 'stats', - $id, - 'value', - $value - ); - } - } - } - } - } catch (\Throwable $e) { - console::error(DateTime::now() . ' ' . $projectInternalId . ' ' . $e->getMessage()); - } - } - }); - } -} diff --git a/src/Appwrite/Specification/Format/OpenAPI3.php b/src/Appwrite/Specification/Format/OpenAPI3.php index b405ec6d1..4bbfce17c 100644 --- a/src/Appwrite/Specification/Format/OpenAPI3.php +++ b/src/Appwrite/Specification/Format/OpenAPI3.php @@ -302,7 +302,7 @@ class OpenAPI3 extends Format case 'Utopia\Database\Validator\UID': case 'Utopia\Validator\Text': $node['schema']['type'] = $validator->getType(); - $node['schema']['x-example'] = '[' . \strtoupper(Template::fromCamelCaseToSnake($node['name'])) . ']'; + $node['schema']['x-example'] = '<' . \strtoupper(Template::fromCamelCaseToSnake($node['name'])) . '>'; break; case 'Utopia\Validator\Boolean': $node['schema']['type'] = $validator->getType(); @@ -313,7 +313,7 @@ class OpenAPI3 extends Format $node['schema']['x-upload-id'] = true; } $node['schema']['type'] = $validator->getType(); - $node['schema']['x-example'] = '[' . \strtoupper(Template::fromCamelCaseToSnake($node['name'])) . ']'; + $node['schema']['x-example'] = '<' . \strtoupper(Template::fromCamelCaseToSnake($node['name'])) . '>'; break; case 'Utopia\Database\Validator\DatetimeValidator': $node['schema']['type'] = $validator->getType(); diff --git a/src/Appwrite/Specification/Format/Swagger2.php b/src/Appwrite/Specification/Format/Swagger2.php index b2ccef6c0..6d0b94cba 100644 --- a/src/Appwrite/Specification/Format/Swagger2.php +++ b/src/Appwrite/Specification/Format/Swagger2.php @@ -298,7 +298,7 @@ class Swagger2 extends Format case 'Utopia\Validator\Text': case 'Utopia\Database\Validator\UID': $node['type'] = $validator->getType(); - $node['x-example'] = '[' . \strtoupper(Template::fromCamelCaseToSnake($node['name'])) . ']'; + $node['x-example'] = '<' . \strtoupper(Template::fromCamelCaseToSnake($node['name'])) . '>'; break; case 'Utopia\Validator\Boolean': $node['type'] = $validator->getType(); @@ -309,7 +309,7 @@ class Swagger2 extends Format $node['x-upload-id'] = true; } $node['type'] = $validator->getType(); - $node['x-example'] = '[' . \strtoupper(Template::fromCamelCaseToSnake($node['name'])) . ']'; + $node['x-example'] = '<' . \strtoupper(Template::fromCamelCaseToSnake($node['name'])) . '>'; break; case 'Utopia\Database\Validator\DatetimeValidator': $node['type'] = $validator->getType(); diff --git a/tests/e2e/Services/Account/AccountConsoleClientTest.php b/tests/e2e/Services/Account/AccountConsoleClientTest.php index 4aa2462f8..97a337453 100644 --- a/tests/e2e/Services/Account/AccountConsoleClientTest.php +++ b/tests/e2e/Services/Account/AccountConsoleClientTest.php @@ -5,10 +5,87 @@ namespace Tests\E2E\Services\Account; use Tests\E2E\Scopes\Scope; use Tests\E2E\Scopes\ProjectConsole; use Tests\E2E\Scopes\SideClient; +use Tests\E2E\Client; +use Utopia\Database\Helpers\ID; class AccountConsoleClientTest extends Scope { use AccountBase; use ProjectConsole; use SideClient; + + public function testDeleteAccount(): void + { + $email = uniqid() . 'user@localhost.test'; + $password = 'password'; + $name = 'User Name'; + + $response = $this->client->call(Client::METHOD_POST, '/account', array_merge([ + 'origin' => 'http://localhost', + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ]), [ + 'userId' => ID::unique(), + 'email' => $email, + 'password' => $password, + 'name' => $name, + ]); + + $this->assertEquals($response['headers']['status-code'], 201); + + $response = $this->client->call(Client::METHOD_POST, '/account/sessions/email', array_merge([ + 'origin' => 'http://localhost', + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ]), [ + 'email' => $email, + 'password' => $password, + ]); + + $this->assertEquals($response['headers']['status-code'], 201); + + $session = $response['cookies']['a_session_' . $this->getProject()['$id']]; + + // create team + $team = $this->client->call(Client::METHOD_POST, '/teams', [ + 'origin' => 'http://localhost', + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'cookie' => 'a_session_' . $this->getProject()['$id'] . '=' . $session, + ], [ + 'teamId' => 'unique()', + 'name' => 'myteam' + ]); + $this->assertEquals($team['headers']['status-code'], 201); + + $teamId = $team['body']['$id']; + + $response = $this->client->call(Client::METHOD_DELETE, '/account', array_merge([ + 'origin' => 'http://localhost', + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'cookie' => 'a_session_' . $this->getProject()['$id'] . '=' . $session, + ])); + + $this->assertEquals($response['headers']['status-code'], 400); + + // DELETE TEAM + $response = $this->client->call(Client::METHOD_DELETE, '/teams/' . $teamId, array_merge([ + 'origin' => 'http://localhost', + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'cookie' => 'a_session_' . $this->getProject()['$id'] . '=' . $session, + ])); + $this->assertEquals($response['headers']['status-code'], 204); + sleep(2); + + $response = $this->client->call(Client::METHOD_DELETE, '/account', array_merge([ + 'origin' => 'http://localhost', + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'cookie' => 'a_session_' . $this->getProject()['$id'] . '=' . $session, + ])); + + $this->assertEquals($response['headers']['status-code'], 204); + } }