diff --git a/app/config/collections.php b/app/config/collections.php index 4d3544f96..3c61dd4e9 100644 --- a/app/config/collections.php +++ b/app/config/collections.php @@ -34,6 +34,28 @@ $commonCollections = [ 'array' => false, 'filters' => [], ], + [ + '$id' => 'resourceType', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 255, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => ID::custom('mimeType'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 255, // https://tools.ietf.org/html/rfc4288#section-4.2 + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], [ '$id' => 'accessedAt', 'type' => Database::VAR_DATETIME, diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index d038b6144..a03784d02 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -2237,7 +2237,7 @@ App::get('/v1/account/logs') } $response->dynamic(new Document([ - 'total' => $audit->countLogsByUser($user->getId()), + 'total' => $audit->countLogsByUser($user->getInternalId()), 'logs' => $output, ]), Response::MODEL_LOG_LIST); }); diff --git a/app/controllers/api/avatars.php b/app/controllers/api/avatars.php index b39335508..6304482b1 100644 --- a/app/controllers/api/avatars.php +++ b/app/controllers/api/avatars.php @@ -76,7 +76,7 @@ $getUserGitHub = function (string $userId, Document $project, Database $dbForPro } if (empty($gitHubSession)) { - throw new Exception(Exception::GENERAL_UNKNOWN, 'GitHub session not found.'); + throw new Exception(Exception::USER_SESSION_NOT_FOUND, 'GitHub session not found.'); } $provider = $gitHubSession->getAttribute('provider', ''); diff --git a/app/controllers/api/health.php b/app/controllers/api/health.php index 274901341..a028b0b59 100644 --- a/app/controllers/api/health.php +++ b/app/controllers/api/health.php @@ -659,6 +659,60 @@ App::get('/v1/health/queue/functions') $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); }, ['response']); +App::get('/v1/health/queue/usage') + ->desc('Get usage queue') + ->groups(['api', 'health']) + ->label('scope', 'health.read') + ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) + ->label('sdk.namespace', 'health') + ->label('sdk.method', 'getQueueUsage') + ->label('sdk.description', '/docs/references/health/get-queue-usage.md') + ->label('sdk.response.code', Response::STATUS_CODE_OK) + ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) + ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE) + ->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true) + ->inject('queue') + ->inject('response') + ->action(function (int|string $threshold, Connection $queue, Response $response) { + $threshold = \intval($threshold); + + $client = new Client(Event::USAGE_QUEUE_NAME, $queue); + $size = $client->getQueueSize(); + + if ($size >= $threshold) { + throw new Exception(Exception::HEALTH_QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); + } + + $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); + }); + +App::get('/v1/health/queue/usage-dump') + ->desc('Get usage dump queue') + ->groups(['api', 'health']) + ->label('scope', 'health.read') + ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) + ->label('sdk.namespace', 'health') + ->label('sdk.method', 'getQueueUsage') + ->label('sdk.description', '/docs/references/health/get-queue-usage-dump.md') + ->label('sdk.response.code', Response::STATUS_CODE_OK) + ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) + ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE) + ->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true) + ->inject('queue') + ->inject('response') + ->action(function (int|string $threshold, Connection $queue, Response $response) { + $threshold = \intval($threshold); + + $client = new Client(Event::USAGE_DUMP_QUEUE_NAME, $queue); + $size = $client->getQueueSize(); + + if ($size >= $threshold) { + throw new Exception(Exception::HEALTH_QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); + } + + $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); + }); + App::get('/v1/health/storage/local') ->desc('Get local storage') ->groups(['api', 'health']) @@ -796,6 +850,7 @@ App::get('/v1/health/queue/failed/:name') Event::MAILS_QUEUE_NAME, Event::FUNCTIONS_QUEUE_NAME, Event::USAGE_QUEUE_NAME, + Event::USAGE_DUMP_QUEUE_NAME, Event::WEBHOOK_CLASS_NAME, Event::CERTIFICATES_QUEUE_NAME, Event::BUILDS_QUEUE_NAME, diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index 62b0ef64c..0574f9c57 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -1481,6 +1481,7 @@ App::delete('/v1/storage/buckets/:bucketId/files/:fileId') if ($deviceDeleted) { $queueForDeletes ->setType(DELETE_TYPE_CACHE_BY_RESOURCE) + ->setResourceType('bucket/' . $bucket->getId()) ->setResource('file/' . $fileId) ; diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index d29712e00..46aef5705 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -813,6 +813,9 @@ App::get('/v1/users/:userId/logs') $output[$i] = new Document([ 'event' => $log['event'], + 'userId' => ID::custom($log['data']['userId']), + 'userEmail' => $log['data']['userEmail'] ?? null, + 'userName' => $log['data']['userName'] ?? null, 'ip' => $log['ip'], 'time' => $log['time'], 'osCode' => $os['osCode'], @@ -841,7 +844,7 @@ App::get('/v1/users/:userId/logs') } $response->dynamic(new Document([ - 'total' => $audit->countLogsByUser($user->getId()), + 'total' => $audit->countLogsByUser($user->getInternalId()), 'logs' => $output, ]), Response::MODEL_LOG_LIST); }); diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 449ca9c40..d55c085f7 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -404,24 +404,22 @@ App::init() ; $useCache = $route->getLabel('cache', false); - if ($useCache) { $key = md5($request->getURI() . implode('*', $request->getParams()) . '*' . APP_CACHE_BUSTER); + $cacheLog = Authorization::skip(fn () => $dbForProject->getDocument('cache', $key)); $cache = new Cache( new Filesystem(APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $project->getId()) ); $timestamp = 60 * 60 * 24 * 30; $data = $cache->load($key, $timestamp); - if (!empty($data)) { - $data = json_decode($data, true); - $parts = explode('/', $data['resourceType']); + if (!empty($data) && !$cacheLog->isEmpty()) { + $parts = explode('/', $cacheLog->getAttribute('resourceType')); $type = $parts[0] ?? null; if ($type === 'bucket') { $bucketId = $parts[1] ?? null; - - $bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); + $bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); $isAPIKey = Auth::isAppUser(Authorization::getRoles()); $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles()); @@ -433,11 +431,12 @@ App::init() $fileSecurity = $bucket->getAttribute('fileSecurity', false); $validator = new Authorization(Database::PERMISSION_READ); $valid = $validator->isValid($bucket->getRead()); + if (!$fileSecurity && !$valid) { throw new Exception(Exception::USER_UNAUTHORIZED); } - $parts = explode('/', $data['resource']); + $parts = explode('/', $cacheLog->getAttribute('resource')); $fileId = $parts[1] ?? null; if ($fileSecurity && !$valid) { @@ -454,8 +453,8 @@ App::init() $response ->addHeader('Expires', \date('D, d M Y H:i:s', \time() + $timestamp) . ' GMT') ->addHeader('X-Appwrite-Cache', 'hit') - ->setContentType($data['contentType']) - ->send(base64_decode($data['payload'])) + ->setContentType($cacheLog->getAttribute('mimeType')) + ->send($data) ; } else { $response->addHeader('X-Appwrite-Cache', 'miss'); @@ -651,7 +650,6 @@ App::shutdown() if ($useCache) { $resource = $resourceType = null; $data = $response->getPayload(); - if (!empty($data['payload'])) { $pattern = $route->getLabel('cache.resource', null); if (!empty($pattern)) { @@ -663,22 +661,17 @@ App::shutdown() $resourceType = $parseLabel($pattern, $responsePayload, $requestParams, $user); } - $key = md5($request->getURI() . implode('*', $request->getParams()) . '*' . APP_CACHE_BUSTER); - $data = json_encode([ - 'resourceType' => $resourceType, - 'resource' => $resource, - 'contentType' => $response->getContentType(), - 'payload' => base64_encode($data['payload']), - ]) ; - - $signature = md5($data); - $cacheLog = Authorization::skip(fn () => $dbForProject->getDocument('cache', $key)); + $key = md5($request->getURI() . '*' . implode('*', $request->getParams())) . '*' . APP_CACHE_BUSTER; + $signature = md5($data['payload']); + $cacheLog = Authorization::skip(fn () => $dbForProject->getDocument('cache', $key)); $accessedAt = $cacheLog->getAttribute('accessedAt', ''); $now = DateTime::now(); if ($cacheLog->isEmpty()) { Authorization::skip(fn () => $dbForProject->createDocument('cache', new Document([ '$id' => $key, 'resource' => $resource, + 'resourceType' => $resourceType, + 'mimeType' => $response->getContentType(), 'accessedAt' => $now, 'signature' => $signature, ]))); @@ -691,7 +684,7 @@ App::shutdown() $cache = new Cache( new Filesystem(APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $project->getId()) ); - $cache->save($key, $data); + $cache->save($key, $data['payload']); } } } diff --git a/docker-compose.yml b/docker-compose.yml index b706dc945..58dff5690 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -971,6 +971,7 @@ services: # - appwrite # volumes: # - appwrite-uploads:/storage/uploads + # Dev Tools Start ------------------------------------------------------------------------------------------ # # The Appwrite Team uses the following tools to help debug, monitor and diagnose the Appwrite stack diff --git a/docs/references/health/get-queue-usage-dump.md b/docs/references/health/get-queue-usage-dump.md new file mode 100644 index 000000000..3c95da1b8 --- /dev/null +++ b/docs/references/health/get-queue-usage-dump.md @@ -0,0 +1 @@ +Get the number of projects containing metrics that are waiting to be processed in the Appwrite internal queue server. \ No newline at end of file diff --git a/docs/references/health/get-queue-usage.md b/docs/references/health/get-queue-usage.md new file mode 100644 index 000000000..8e5b64e64 --- /dev/null +++ b/docs/references/health/get-queue-usage.md @@ -0,0 +1 @@ +Get the number of metrics that are waiting to be processed in the Appwrite internal queue server. \ No newline at end of file diff --git a/src/Appwrite/Event/Delete.php b/src/Appwrite/Event/Delete.php index 57300feb7..064fbcefa 100644 --- a/src/Appwrite/Event/Delete.php +++ b/src/Appwrite/Event/Delete.php @@ -10,6 +10,7 @@ class Delete extends Event { protected string $type = ''; protected ?Document $document = null; + protected ?string $resourceType = null; protected ?string $resource = null; protected ?string $datetime = null; protected ?string $hourlyUsageRetentionDatetime = null; @@ -107,6 +108,19 @@ class Delete extends Event return $this; } + /** + * Sets the resource type for the delete event. + * + * @param string $resourceType + * @return self + */ + public function setResourceType(string $resourceType): self + { + $this->resourceType = $resourceType; + + return $this; + } + /** * Returns the set document for the delete event. * @@ -133,6 +147,7 @@ class Delete extends Event 'type' => $this->type, 'document' => $this->document, 'resource' => $this->resource, + 'resourceType' => $this->resourceType, 'datetime' => $this->datetime, 'hourlyUsageRetentionDatetime' => $this->hourlyUsageRetentionDatetime ]); diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 920496055..fdf494afe 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -71,6 +71,7 @@ class Deletes extends Action $datetime = $payload['datetime'] ?? null; $hourlyUsageRetentionDatetime = $payload['hourlyUsageRetentionDatetime'] ?? null; $resource = $payload['resource'] ?? null; + $resourceType = $payload['resourceType'] ?? null; $document = new Document($payload['document'] ?? []); $project = new Document($payload['project'] ?? []); @@ -80,12 +81,6 @@ class Deletes extends Action switch (\strval($type)) { case DELETE_TYPE_DOCUMENT: switch ($document->getCollection()) { - case DELETE_TYPE_DATABASES: - $this->deleteDatabase($getProjectDB, $document, $project); - break; - case DELETE_TYPE_COLLECTIONS: - $this->deleteCollection($getProjectDB, $document, $project); - break; case DELETE_TYPE_PROJECTS: $this->deleteProject($dbForConsole, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $document); break; @@ -114,10 +109,6 @@ class Deletes extends Action $this->deleteRule($dbForConsole, $document); break; default: - if (\str_starts_with($document->getCollection(), 'database_')) { - $this->deleteCollection($getProjectDB, $document, $project); - break; - } Console::error('No lazy delete operation available for document of type: ' . $document->getCollection()); break; } @@ -147,7 +138,7 @@ class Deletes extends Action $this->deleteUsageStats($project, $getProjectDB, $hourlyUsageRetentionDatetime); break; case DELETE_TYPE_CACHE_BY_RESOURCE: - $this->deleteCacheByResource($project, $getProjectDB, $resource); + $this->deleteCacheByResource($project, $getProjectDB, $resource, $resourceType); break; case DELETE_TYPE_CACHE_BY_TIMESTAMP: $this->deleteCacheByDate($project, $getProjectDB, $datetime); @@ -332,32 +323,37 @@ class Deletes extends Action * @param string $resource * @return void * @throws Authorization + * @param string|null $resourceType + * @throws Exception */ - private function deleteCacheByResource(Document $project, callable $getProjectDB, string $resource): void + private function deleteCacheByResource(Document $project, callable $getProjectDB, string $resource, string $resourceType = null): void { $projectId = $project->getId(); $dbForProject = $getProjectDB($project); - $document = $dbForProject->findOne('cache', [Query::equal('resource', [$resource])]); - if ($document) { - $cache = new Cache( - new Filesystem(APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $projectId) - ); + $cache = new Cache( + new Filesystem(APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $projectId) + ); - $this->deleteById( - $document, - $dbForProject, - function ($document) use ($cache, $projectId) { - $path = APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $projectId . DIRECTORY_SEPARATOR . $document->getId(); - - if ($cache->purge($document->getId())) { - Console::success('Deleting cache file: ' . $path); - } else { - Console::error('Failed to delete cache file: ' . $path); - } - } - ); + $query[] = Query::equal('resource', [$resource]); + if (!empty($resourceType)) { + $query[] = Query::equal('resourceType', [$resourceType]); } + + $this->deleteByGroup( + 'cache', + $query, + $dbForProject, + function (Document $document) use ($cache, $projectId) { + $path = APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $projectId . DIRECTORY_SEPARATOR . $document->getId(); + + if ($cache->purge($document->getId())) { + Console::success('Deleting cache file: ' . $path); + } else { + Console::error('Failed to delete cache file: ' . $path); + } + } + ); } /** @@ -397,72 +393,6 @@ class Deletes extends Action ); } - /** - * @param callable $getProjectDB - * @param Document $document - * @param Document $project - * @return void - * @throws Exception - */ - private function deleteDatabase(callable $getProjectDB, Document $document, Document $project): void - { - $databaseId = $document->getId(); - $dbForProject = $getProjectDB($project); - - $this->deleteByGroup('database_' . $document->getInternalId(), [], $dbForProject, function ($document) use ($getProjectDB, $project) { - $this->deleteCollection($getProjectDB, $document, $project); - }); - - $dbForProject->deleteCollection('database_' . $document->getInternalId()); - $this->deleteAuditLogsByResource($getProjectDB, 'database/' . $databaseId, $project); - } - - /** - * @param callable $getProjectDB - * @param Document $document teams document - * @param Document $project - * @return void - * @throws Exception - */ - private function deleteCollection(callable $getProjectDB, Document $document, Document $project): void - { - $collectionId = $document->getId(); - $collectionInternalId = $document->getInternalId(); - $databaseId = $document->getAttribute('databaseId'); - $databaseInternalId = $document->getAttribute('databaseInternalId'); - - $dbForProject = $getProjectDB($project); - - $relationships = \array_filter( - $document->getAttribute('attributes'), - fn ($attribute) => $attribute['type'] === Database::VAR_RELATIONSHIP - ); - - foreach ($relationships as $relationship) { - if (!$relationship['twoWay']) { - continue; - } - $relatedCollection = $dbForProject->getDocument('database_' . $databaseInternalId, $relationship['relatedCollection']); - $dbForProject->deleteDocument('attributes', $databaseInternalId . '_' . $relatedCollection->getInternalId() . '_' . $relationship['twoWayKey']); - $dbForProject->deleteCachedDocument('database_' . $databaseInternalId, $relatedCollection->getId()); - $dbForProject->deleteCachedCollection('database_' . $databaseInternalId . '_collection_' . $relatedCollection->getInternalId()); - } - - $dbForProject->deleteCollection('database_' . $databaseInternalId . '_collection_' . $document->getInternalId()); - - $this->deleteByGroup('attributes', [ - Query::equal('databaseInternalId', [$databaseInternalId]), - Query::equal('collectionInternalId', [$collectionInternalId]) - ], $dbForProject); - - $this->deleteByGroup('indexes', [ - Query::equal('databaseInternalId', [$databaseInternalId]), - Query::equal('collectionInternalId', [$collectionInternalId]) - ], $dbForProject); - - $this->deleteAuditLogsByResource($getProjectDB, 'database/' . $databaseId . '/collection/' . $collectionId, $project); - } - /** * @param Database $dbForConsole * @param callable $getProjectDB diff --git a/src/Appwrite/Specification/Format/OpenAPI3.php b/src/Appwrite/Specification/Format/OpenAPI3.php index bc85d8e50..a2539f30b 100644 --- a/src/Appwrite/Specification/Format/OpenAPI3.php +++ b/src/Appwrite/Specification/Format/OpenAPI3.php @@ -291,7 +291,7 @@ class OpenAPI3 extends Format case 'Utopia\Database\Validator\UID': case 'Utopia\Validator\Text': $node['schema']['type'] = $validator->getType(); - $node['schema']['x-example'] = '[' . \strtoupper(Template::fromCamelCaseToSnake($node['name'])) . ']'; + $node['schema']['x-example'] = '<' . \strtoupper(Template::fromCamelCaseToSnake($node['name'])) . '>'; break; case 'Utopia\Validator\Boolean': $node['schema']['type'] = $validator->getType(); @@ -302,7 +302,7 @@ class OpenAPI3 extends Format $node['schema']['x-upload-id'] = true; } $node['schema']['type'] = $validator->getType(); - $node['schema']['x-example'] = '[' . \strtoupper(Template::fromCamelCaseToSnake($node['name'])) . ']'; + $node['schema']['x-example'] = '<' . \strtoupper(Template::fromCamelCaseToSnake($node['name'])) . '>'; break; case 'Utopia\Database\Validator\DatetimeValidator': $node['schema']['type'] = $validator->getType(); diff --git a/src/Appwrite/Specification/Format/Swagger2.php b/src/Appwrite/Specification/Format/Swagger2.php index f923741b4..dd2b12b6b 100644 --- a/src/Appwrite/Specification/Format/Swagger2.php +++ b/src/Appwrite/Specification/Format/Swagger2.php @@ -290,7 +290,7 @@ class Swagger2 extends Format case 'Utopia\Validator\Text': case 'Utopia\Database\Validator\UID': $node['type'] = $validator->getType(); - $node['x-example'] = '[' . \strtoupper(Template::fromCamelCaseToSnake($node['name'])) . ']'; + $node['x-example'] = '<' . \strtoupper(Template::fromCamelCaseToSnake($node['name'])) . '>'; break; case 'Utopia\Validator\Boolean': $node['type'] = $validator->getType(); @@ -301,7 +301,7 @@ class Swagger2 extends Format $node['x-upload-id'] = true; } $node['type'] = $validator->getType(); - $node['x-example'] = '[' . \strtoupper(Template::fromCamelCaseToSnake($node['name'])) . ']'; + $node['x-example'] = '<' . \strtoupper(Template::fromCamelCaseToSnake($node['name'])) . '>'; break; case 'Utopia\Database\Validator\DatetimeValidator': $node['type'] = $validator->getType(); diff --git a/tests/e2e/Services/Health/HealthCustomServerTest.php b/tests/e2e/Services/Health/HealthCustomServerTest.php index e9fe4b4f8..5fd7ab96d 100644 --- a/tests/e2e/Services/Health/HealthCustomServerTest.php +++ b/tests/e2e/Services/Health/HealthCustomServerTest.php @@ -511,4 +511,52 @@ class HealthCustomServerTest extends Scope return []; } + + public function testUsageSuccess() + { + /** + * Test for SUCCESS + */ + $response = $this->client->call(Client::METHOD_GET, '/health/queue/usage', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), []); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertIsInt($response['body']['size']); + $this->assertLessThan(100, $response['body']['size']); + + /** + * Test for FAILURE + */ + $response = $this->client->call(Client::METHOD_GET, '/health/queue/usage?threshold=0', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), []); + $this->assertEquals(503, $response['headers']['status-code']); + } + + public function testUsageDumpSuccess() + { + /** + * Test for SUCCESS + */ + $response = $this->client->call(Client::METHOD_GET, '/health/queue/usage-dump', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), []); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertIsInt($response['body']['size']); + $this->assertLessThan(100, $response['body']['size']); + + /** + * Test for FAILURE + */ + $response = $this->client->call(Client::METHOD_GET, '/health/queue/usage-dump?threshold=0', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), []); + $this->assertEquals(503, $response['headers']['status-code']); + } }