diff --git a/.env b/.env index 709a761d2..227ea1067 100644 --- a/.env +++ b/.env @@ -72,6 +72,7 @@ OPEN_RUNTIMES_NETWORK=appwrite_runtimes _APP_EXECUTOR_SECRET=your-secret-key _APP_EXECUTOR_HOST=http://appwrite-executor/v1 _APP_MAINTENANCE_INTERVAL=86400 +_APP_MAINTENANCE_RETENTION_CACHE=2592000 _APP_MAINTENANCE_RETENTION_EXECUTION=1209600 _APP_MAINTENANCE_RETENTION_ABUSE=86400 _APP_MAINTENANCE_RETENTION_AUDIT=1209600 diff --git a/.gitignore b/.gitignore index 3d6001ca0..5484542dd 100644 --- a/.gitignore +++ b/.gitignore @@ -9,4 +9,4 @@ .php_cs.cache debug/ app/sdks -dev/yasd_init.php +dev/yasd_init.php \ No newline at end of file diff --git a/app/config/collections.php b/app/config/collections.php index 375b2a945..bbc66cbb8 100644 --- a/app/config/collections.php +++ b/app/config/collections.php @@ -2798,6 +2798,62 @@ $collections = [ ], ] ], + 'cache' => [ + '$collection' => Database::METADATA, + '$id' => 'cache', + 'name' => 'Cache', + 'attributes' => [ + [ + '$id' => 'resource', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 255, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'accessedAt', + 'type' => Database::VAR_INTEGER, + 'format' => '', + 'size' => 0, + 'signed' => false, + 'required' => true, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'signature', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 255, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + ], + 'indexes' => [ + [ + '$id' => '_key_accessedAt', + 'type' => Database::INDEX_KEY, + 'attributes' => ['accessedAt'], + 'lengths' => [], + 'orders' => [], + ], + [ + '$id' => '_key_resource', + 'type' => Database::INDEX_KEY, + 'attributes' => ['resource'], + 'lengths' => [], + 'orders' => [], + ], + ], + ], 'files' => [ '$collection' => 'buckets', '$id' => 'files', diff --git a/app/config/variables.php b/app/config/variables.php index 9b2decaf5..9ca42c3e8 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -822,6 +822,15 @@ return [ 'question' => '', 'filter' => '' ], + [ + 'name' => '_APP_MAINTENANCE_RETENTION_CACHE', + 'description' => 'The maximum duration (in seconds) upto which to retain cached files. The default value is 2592000 seconds (30 days).', + 'introduction' => '0.16.0', + 'default' => '2592000', + 'required' => false, + 'question' => '', + 'filter' => '' + ], [ 'name' => '_APP_MAINTENANCE_RETENTION_EXECUTION', 'description' => 'The maximum duration (in seconds) upto which to retain execution logs. The default value is 1209600 seconds (14 days).', diff --git a/app/controllers/api/avatars.php b/app/controllers/api/avatars.php index afdd7d4da..6dc31787b 100644 --- a/app/controllers/api/avatars.php +++ b/app/controllers/api/avatars.php @@ -7,8 +7,6 @@ use Appwrite\Utopia\Response; use chillerlan\QRCode\QRCode; use chillerlan\QRCode\QROptions; use Utopia\App; -use Utopia\Cache\Adapter\Filesystem; -use Utopia\Cache\Cache; use Utopia\Config\Config; use Utopia\Database\Document; use Utopia\Image\Image; @@ -37,8 +35,6 @@ $avatarCallback = function (string $type, string $code, int $width, int $height, } $output = 'png'; - $date = \date('D, d M Y H:i:s', \time() + (60 * 60 * 24 * 45)) . ' GMT'; // 45 days cache - $key = \md5('/v1/avatars/' . $type . '/:code-' . $code . $width . $height . $quality . $output); $path = $set[$code]; $type = 'png'; @@ -46,35 +42,15 @@ $avatarCallback = function (string $type, string $code, int $width, int $height, throw new Exception(Exception::GENERAL_SERVER_ERROR, 'File not readable in ' . $path); } - $cache = new Cache(new Filesystem(APP_STORAGE_CACHE . '/app-0')); // Limit file number or size - $data = $cache->load($key, 60 * 60 * 24 * 30 * 3/* 3 months */); - - if ($data) { - //$output = (empty($output)) ? $type : $output; - - return $response - ->setContentType('image/png') - ->addHeader('Expires', $date) - ->addHeader('X-Appwrite-Cache', 'hit') - ->send($data); - } - $image = new Image(\file_get_contents($path)); - $image->crop((int) $width, (int) $height); - $output = (empty($output)) ? $type : $output; - $data = $image->output($output, $quality); - - $cache->save($key, $data); - $response + ->addHeader('Expires', \date('D, d M Y H:i:s', \time() + 60 * 60 * 24 * 30) . ' GMT') ->setContentType('image/png') - ->addHeader('Expires', $date) - ->addHeader('X-Appwrite-Cache', 'miss') - ->send($data, null); - + ->file($data) + ; unset($image); }; @@ -82,6 +58,8 @@ App::get('/v1/avatars/credit-cards/:code') ->desc('Get Credit Card Icon') ->groups(['api', 'avatars']) ->label('scope', 'avatars.read') + ->label('cache', true) + ->label('cache.resource', 'avatar/credit-card') ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT]) ->label('sdk.namespace', 'avatars') ->label('sdk.method', 'getCreditCard') @@ -100,6 +78,8 @@ App::get('/v1/avatars/browsers/:code') ->desc('Get Browser Icon') ->groups(['api', 'avatars']) ->label('scope', 'avatars.read') + ->label('cache', true) + ->label('cache.resource', 'avatar/browser') ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT]) ->label('sdk.namespace', 'avatars') ->label('sdk.method', 'getBrowser') @@ -118,6 +98,8 @@ App::get('/v1/avatars/flags/:code') ->desc('Get Country Flag') ->groups(['api', 'avatars']) ->label('scope', 'avatars.read') + ->label('cache', true) + ->label('cache.resource', 'avatar/flag') ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT]) ->label('sdk.namespace', 'avatars') ->label('sdk.method', 'getFlag') @@ -136,6 +118,8 @@ App::get('/v1/avatars/image') ->desc('Get Image from URL') ->groups(['api', 'avatars']) ->label('scope', 'avatars.read') + ->label('cache', true) + ->label('cache.resource', 'avatar/image') ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT]) ->label('sdk.namespace', 'avatars') ->label('sdk.method', 'getImage') @@ -151,19 +135,7 @@ App::get('/v1/avatars/image') $quality = 80; $output = 'png'; - $date = \date('D, d M Y H:i:s', \time() + (60 * 60 * 24 * 45)) . ' GMT'; // 45 days cache - $key = \md5('/v2/avatars/images-' . $url . '-' . $width . '/' . $height . '/' . $quality); $type = 'png'; - $cache = new Cache(new Filesystem(APP_STORAGE_CACHE . '/app-0')); // Limit file number or size - $data = $cache->load($key, 60 * 60 * 24 * 7/* 1 week */); - - if ($data) { - return $response - ->setContentType('image/png') - ->addHeader('Expires', $date) - ->addHeader('X-Appwrite-Cache', 'hit') - ->send($data); - } if (!\extension_loaded('imagick')) { throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Imagick extension is missing'); @@ -182,19 +154,14 @@ App::get('/v1/avatars/image') } $image->crop((int) $width, (int) $height); - $output = (empty($output)) ? $type : $output; - $data = $image->output($output, $quality); - $cache->save($key, $data); - $response + ->addHeader('Expires', \date('D, d M Y H:i:s', \time() + 60 * 60 * 24 * 30) . ' GMT') ->setContentType('image/png') - ->addHeader('Expires', $date) - ->addHeader('X-Appwrite-Cache', 'miss') - ->send($data); - + ->file($data) + ; unset($image); }); @@ -202,6 +169,8 @@ App::get('/v1/avatars/favicon') ->desc('Get Favicon') ->groups(['api', 'avatars']) ->label('scope', 'avatars.read') + ->label('cache', true) + ->label('cache.resource', 'avatar/favicon') ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT]) ->label('sdk.namespace', 'avatars') ->label('sdk.method', 'getFavicon') @@ -217,19 +186,7 @@ App::get('/v1/avatars/favicon') $height = 56; $quality = 80; $output = 'png'; - $date = \date('D, d M Y H:i:s', \time() + (60 * 60 * 24 * 45)) . ' GMT'; // 45 days cache - $key = \md5('/v2/avatars/favicon-' . $url); $type = 'png'; - $cache = new Cache(new Filesystem(APP_STORAGE_CACHE . '/app-0')); // Limit file number or size - $data = $cache->load($key, 60 * 60 * 24 * 30 * 3/* 3 months */); - - if ($data) { - return $response - ->setContentType('image/png') - ->addHeader('Expires', $date) - ->addHeader('X-Appwrite-Cache', 'hit') - ->send($data); - } if (!\extension_loaded('imagick')) { throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Imagick extension is missing'); @@ -314,14 +271,11 @@ App::get('/v1/avatars/favicon') if (empty($data) || (\mb_substr($data, 0, 5) === 'save($key, $data); - - return $response + $response + ->addHeader('Expires', \date('D, d M Y H:i:s', \time() + 60 * 60 * 24 * 30) . ' GMT') ->setContentType('image/x-icon') - ->addHeader('Expires', $date) - ->addHeader('X-Appwrite-Cache', 'miss') - ->send($data); + ->file($data) + ; } $fetch = @\file_get_contents($outputHref, false); @@ -331,21 +285,15 @@ App::get('/v1/avatars/favicon') } $image = new Image($fetch); - $image->crop((int) $width, (int) $height); - $output = (empty($output)) ? $type : $output; - $data = $image->output($output, $quality); - $cache->save($key, $data); - $response + ->addHeader('Expires', \date('D, d M Y H:i:s', \time() + 60 * 60 * 24 * 30) . ' GMT') ->setContentType('image/png') - ->addHeader('Expires', $date) - ->addHeader('X-Appwrite-Cache', 'miss') - ->send($data); - + ->file($data) + ; unset($image); }); @@ -381,19 +329,21 @@ App::get('/v1/avatars/qr') } $image = new Image($qrcode->render($text)); - $image->crop((int) $size, (int) $size); $response ->addHeader('Expires', \date('D, d M Y H:i:s', \time() + (60 * 60 * 24 * 45)) . ' GMT') // 45 days cache ->setContentType('image/png') - ->send($image->output('png', 9)); + ->send($image->output('png', 9)) + ; }); App::get('/v1/avatars/initials') ->desc('Get User Initials') ->groups(['api', 'avatars']) ->label('scope', 'avatars.read') + ->label('cache', true) + ->label('cache.resource', 'avatar/initials') ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT]) ->label('sdk.namespace', 'avatars') ->label('sdk.method', 'getInitials') @@ -468,5 +418,6 @@ App::get('/v1/avatars/initials') $response ->addHeader('Expires', \date('D, d M Y H:i:s', \time() + (60 * 60 * 24 * 45)) . ' GMT') // 45 days cache ->setContentType('image/png') - ->send($image->getImageBlob()); + ->file($image->getImageBlob()) + ; }); diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index f8d4df49e..98b880b2a 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -127,6 +127,7 @@ App::post('/v1/projects') if (($collection['$collection'] ?? '') !== Database::METADATA) { continue; } + $attributes = []; $indexes = []; @@ -153,7 +154,6 @@ App::post('/v1/projects') 'orders' => $index['orders'], ]); } - $dbForProject->createCollection($key, $attributes, $indexes); } diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index fa0ef8217..0c31a735f 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -9,8 +9,6 @@ use Appwrite\OpenSSL\OpenSSL; use Appwrite\Usage\Stats; use Appwrite\Utopia\Response; use Utopia\App; -use Utopia\Cache\Adapter\Filesystem; -use Utopia\Cache\Cache; use Utopia\Config\Config; use Utopia\Database\Database; use Utopia\Database\Document; @@ -34,6 +32,7 @@ use Utopia\Storage\Validator\Upload; use Utopia\Validator\ArrayList; use Utopia\Validator\Boolean; use Utopia\Validator\HexColor; +use Utopia\Validator\Integer; use Utopia\Validator\Range; use Utopia\Validator\Text; use Utopia\Validator\WhiteList; @@ -328,7 +327,9 @@ App::post('/v1/storage/buckets/:bucketId/files') ->inject('mode') ->inject('deviceFiles') ->inject('deviceLocal') - ->action(function (string $bucketId, string $fileId, mixed $file, ?array $read, ?array $write, Request $request, Response $response, Database $dbForProject, Document $user, Event $events, string $mode, Device $deviceFiles, Device $deviceLocal) { + ->inject('deletes') + ->action(function (string $bucketId, string $fileId, mixed $file, ?array $read, ?array $write, Request $request, Response $response, Database $dbForProject, Document $user, Event $events, string $mode, Device $deviceFiles, Device $deviceLocal, Delete $deletes) { + $bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); if ( @@ -617,6 +618,11 @@ App::post('/v1/storage/buckets/:bucketId/files') ->setContext('bucket', $bucket) ; + $deletes + ->setType(DELETE_TYPE_CACHE_BY_RESOURCE) + ->setResource('file/' . $file->getId()) + ; + $metadata = null; // was causing leaks as it was passed by reference $response->setStatusCode(Response::STATUS_CODE_CREATED); @@ -758,6 +764,8 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview') ->desc('Get File Preview') ->groups(['api', 'storage']) ->label('scope', 'files.read') + ->label('cache', true) + ->label('cache.resource', 'file/{request.fileId}') ->label('usage.metric', 'files.{scope}.requests.read') ->label('usage.params', ['bucketId' => 'request.bucketId']) ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT]) @@ -818,9 +826,6 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview') $outputs = Config::getParam('storage-outputs'); $fileLogos = Config::getParam('storage-logos'); - $date = \date('D, d M Y H:i:s', \time() + (60 * 60 * 24 * 45)) . ' GMT'; // 45 days cache - $key = \md5($fileId . $width . $height . $gravity . $quality . $borderWidth . $borderColor . $borderRadius . $opacity . $rotation . $background . $output); - if ($bucket->getAttribute('permission') === 'bucket') { // skip authorization $file = Authorization::skip(fn () => $dbForProject->getDocument('bucket_' . $bucket->getInternalId(), $fileId)); @@ -837,7 +842,6 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview') $algorithm = $file->getAttribute('algorithm'); $cipher = $file->getAttribute('openSSLCipher'); $mime = $file->getAttribute('mimeType'); - if (!\in_array($mime, $inputs) || $file->getAttribute('sizeActual') > (int) App::getEnv('_APP_STORAGE_PREVIEW_LIMIT', 20000000)) { if (!\in_array($mime, $inputs)) { $path = (\array_key_exists($mime, $fileLogos)) ? $fileLogos[$mime] : $fileLogos['default']; @@ -850,7 +854,6 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview') $cipher = null; $background = (empty($background)) ? 'eceff1' : $background; $type = \strtolower(\pathinfo($path, PATHINFO_EXTENSION)); - $key = \md5($path . $width . $height . $gravity . $quality . $borderWidth . $borderColor . $borderRadius . $opacity . $rotation . $background . $output); $deviceFiles = $deviceLocal; } @@ -861,23 +864,12 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview') throw new Exception(Exception::STORAGE_FILE_NOT_FOUND); } - $cache = new Cache(new Filesystem(APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $project->getId() . DIRECTORY_SEPARATOR . $bucketId . DIRECTORY_SEPARATOR . $fileId)); // Limit file number or size - $data = $cache->load($key, 60 * 60 * 24 * 30 * 3/* 3 months */); - if (empty($output)) { // when file extension is not provided and the mime type is not one of our supported outputs // we fallback to `jpg` output format $output = empty($type) ? (array_search($mime, $outputs) ?? 'jpg') : $type; } - if ($data) { - return $response - ->setContentType((\array_key_exists($output, $outputs)) ? $outputs[$output] : $outputs['jpg']) - ->addHeader('Expires', $date) - ->addHeader('X-Appwrite-Cache', 'hit') - ->send($data) - ; - } $source = $deviceFiles->read($path); @@ -922,13 +914,12 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview') $data = $image->output($output, $quality); - $cache->save($key, $data); + $contentType = (\array_key_exists($output, $outputs)) ? $outputs[$output] : $outputs['jpg']; $response - ->setContentType((\array_key_exists($output, $outputs)) ? $outputs[$output] : $outputs['jpg']) - ->addHeader('Expires', $date) - ->addHeader('X-Appwrite-Cache', 'miss') - ->send($data) + ->addHeader('Expires', \date('D, d M Y H:i:s', \time() + 60 * 60 * 24 * 30) . ' GMT') + ->setContentType($contentType) + ->file($data) ; unset($image); @@ -1330,8 +1321,8 @@ App::delete('/v1/storage/buckets/:bucketId/files/:fileId') ->inject('events') ->inject('mode') ->inject('deviceFiles') - ->inject('project') - ->action(function (string $bucketId, string $fileId, Response $response, Database $dbForProject, Event $events, string $mode, Device $deviceFiles, Document $project) { + ->inject('deletes') + ->action(function (string $bucketId, string $fileId, Response $response, Database $dbForProject, Event $events, string $mode, Device $deviceFiles, Delete $deletes) { $bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); if ( @@ -1370,10 +1361,10 @@ App::delete('/v1/storage/buckets/:bucketId/files/:fileId') } if ($deviceDeleted) { - //delete related cache - $cacheDir = APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $project->getId() . DIRECTORY_SEPARATOR . $bucketId . DIRECTORY_SEPARATOR . $fileId; - $deviceLocal = new Local($cacheDir); - $deviceLocal->delete($cacheDir, true); + $deletes + ->setType(DELETE_TYPE_CACHE_BY_RESOURCE) + ->setResource('file/' . $fileId) + ; if ($bucket->getAttribute('permission') === 'bucket') { $deleted = Authorization::skip(fn () => $dbForProject->deleteDocument('bucket_' . $bucket->getInternalId(), $fileId)); diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 7c1ff407a..1c662511b 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -14,6 +14,8 @@ use Utopia\App; use Appwrite\Extend\Exception; use Utopia\Abuse\Abuse; use Utopia\Abuse\Adapters\TimeLimit; +use Utopia\Cache\Adapter\Filesystem; +use Utopia\Cache\Cache; use Utopia\Database\Database; use Utopia\Database\Document; use Utopia\Database\Validator\Authorization; @@ -67,7 +69,7 @@ App::init() throw new Exception(Exception::PROJECT_UNKNOWN); } - /* + /** * Abuse Check */ $abuseKeyLabel = $route->getLabel('abuse-key', 'url:{url},ip:{ip}'); @@ -146,6 +148,31 @@ App::init() $deletes->setProject($project); $database->setProject($project); + + $useCache = $route->getLabel('cache', false); + + if ($useCache) { + $key = md5($request->getURI() . implode('*', $request->getParams())); + $cache = new Cache( + new Filesystem(APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $project->getId()) + ); + $timestamp = 60 * 60 * 24 * 30; + $data = $cache->load($key, $timestamp); + if (!empty($data)) { + $data = json_decode($data, true); + + $response + ->addHeader('Expires', \date('D, d M Y H:i:s', \time() + $timestamp) . ' GMT') + ->addHeader('X-Appwrite-Cache', 'hit') + ->setContentType($data['content-type']) + ->send(base64_decode($data['payload'])) + ; + + $route->setIsActive(false); + } else { + $response->addHeader('X-Appwrite-Cache', 'miss'); + } + } }); App::init() @@ -216,6 +243,7 @@ App::shutdown() ->inject('mode') ->inject('dbForProject') ->action(function (App $utopia, Request $request, Response $response, Document $project, Event $events, Audit $audits, Stats $usage, Delete $deletes, EventDatabase $database, string $mode, Database $dbForProject) use ($parseLabel) { + $responsePayload = $response->getPayload(); if (!empty($events->getEvent())) { @@ -277,6 +305,9 @@ App::shutdown() $requestParams = $route->getParamsValues(); $user = $audits->getUser(); + /** + * Audit labels + */ $pattern = $route->getLabel('audits.resource', null); if (!empty($pattern)) { $resource = $parseLabel($pattern, $responsePayload, $requestParams, $user); @@ -316,6 +347,49 @@ App::shutdown() $database->trigger(); } + /** + * Cache label + */ + $useCache = $route->getLabel('cache', false); + if ($useCache) { + $resource = null; + $data = $response->getPayload(); + if (!empty($data['payload'])) { + $pattern = $route->getLabel('cache.resource', null); + if (!empty($pattern)) { + $resource = $parseLabel($pattern, $responsePayload, $requestParams, $user); + } + + $key = md5($request->getURI() . implode('*', $request->getParams())); + + $data = json_encode([ + 'content-type' => $response->getContentType(), + 'payload' => base64_encode($data['payload']), + ]) ; + + $signature = md5($data); + $cacheLog = $dbForProject->getDocument('cache', $key); + if ($cacheLog->isEmpty()) { + Authorization::skip(fn () => $dbForProject->createDocument('cache', new Document([ + '$id' => $key, + 'resource' => $resource, + 'accessedAt' => \time(), + 'signature' => $signature, + ]))); + } elseif (date('Y/m/d', \time()) > date('Y/m/d', $cacheLog->getAttribute('accessedAt'))) { + $cacheLog->setAttribute('accessedAt', \time()); + Authorization::skip(fn () => $dbForProject->updateDocument('cache', $cacheLog->getId(), $cacheLog)); + } + + if ($signature !== $cacheLog->getAttribute('signature')) { + $cache = new Cache( + new Filesystem(APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $project->getId()) + ); + $cache->save($key, $data); + } + } + } + if ( App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled' && $project->getId() diff --git a/app/init.php b/app/init.php index 3051ba89d..950c4ba4a 100644 --- a/app/init.php +++ b/app/init.php @@ -143,6 +143,8 @@ const DELETE_TYPE_USAGE = 'usage'; const DELETE_TYPE_REALTIME = 'realtime'; const DELETE_TYPE_BUCKETS = 'buckets'; const DELETE_TYPE_SESSIONS = 'sessions'; +const DELETE_TYPE_CACHE_BY_TIMESTAMP = 'cacheByTimeStamp'; +const DELETE_TYPE_CACHE_BY_RESOURCE = 'cacheByResource'; // Mail Types const MAIL_TYPE_VERIFICATION = 'verification'; const MAIL_TYPE_MAGIC_SESSION = 'magicSession'; diff --git a/app/tasks/maintenance.php b/app/tasks/maintenance.php index eb50dbc0e..73df45885 100644 --- a/app/tasks/maintenance.php +++ b/app/tasks/maintenance.php @@ -127,6 +127,15 @@ $cli } } + function notifyDeleteCache($interval) + { + + (new Delete()) + ->setType(DELETE_TYPE_CACHE_BY_TIMESTAMP) + ->setTimestamp(time() - $interval) + ->trigger(); + } + // # of days in seconds (1 day = 86400s) $interval = (int) App::getEnv('_APP_MAINTENANCE_INTERVAL', '86400'); $executionLogsRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_EXECUTION', '1209600'); @@ -134,8 +143,9 @@ $cli $abuseLogsRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_ABUSE', '86400'); $usageStatsRetention30m = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_USAGE_30M', '129600'); //36 hours $usageStatsRetention1d = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_USAGE_1D', '8640000'); // 100 days + $cacheRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_CACHE', '2592000'); // 30 days - Console::loop(function () use ($interval, $executionLogsRetention, $abuseLogsRetention, $auditLogRetention, $usageStatsRetention30m, $usageStatsRetention1d) { + Console::loop(function () use ($interval, $executionLogsRetention, $abuseLogsRetention, $auditLogRetention, $usageStatsRetention30m, $usageStatsRetention1d, $cacheRetention) { $database = getConsoleDB(); $time = date('d-m-Y H:i:s', time()); @@ -147,5 +157,6 @@ $cli notifyDeleteConnections(); notifyDeleteExpiredSessions(); renewCertificates($database); + notifyDeleteCache($cacheRetention); }, $interval); }); diff --git a/app/views/install/compose.phtml b/app/views/install/compose.phtml index d57593e7b..0442d8456 100644 --- a/app/views/install/compose.phtml +++ b/app/views/install/compose.phtml @@ -147,6 +147,7 @@ services: - _APP_STATSD_PORT - _APP_MAINTENANCE_INTERVAL - _APP_MAINTENANCE_RETENTION_EXECUTION + - _APP_MAINTENANCE_RETENTION_CACHE - _APP_MAINTENANCE_RETENTION_ABUSE - _APP_MAINTENANCE_RETENTION_AUDIT - _APP_SMS_PROVIDER @@ -545,6 +546,7 @@ services: - _APP_DB_PASS - _APP_MAINTENANCE_INTERVAL - _APP_MAINTENANCE_RETENTION_EXECUTION + - _APP_MAINTENANCE_RETENTION_CACHE - _APP_MAINTENANCE_RETENTION_ABUSE - _APP_MAINTENANCE_RETENTION_AUDIT diff --git a/app/workers/deletes.php b/app/workers/deletes.php index a4616f54c..15b5b555e 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -1,6 +1,8 @@ args['project'] ?? []); $type = $this->args['type'] ?? ''; - switch (strval($type)) { case DELETE_TYPE_DOCUMENT: $document = new Document($this->args['document'] ?? []); @@ -107,6 +108,13 @@ class DeletesV1 extends Worker case DELETE_TYPE_USAGE: $this->deleteUsageStats($this->args['timestamp1d'], $this->args['timestamp30m']); break; + + case DELETE_TYPE_CACHE_BY_RESOURCE: + $this->deleteCacheByResource($project->getId()); + break; + case DELETE_TYPE_CACHE_BY_TIMESTAMP: + $this->deleteCacheByTimestamp(); + break; default: Console::error('No delete operation for type: ' . $type); break; @@ -117,6 +125,49 @@ class DeletesV1 extends Worker { } + /** + * @param string $projectId + */ + protected function deleteCacheByResource(string $projectId): void + { + $this->deleteCacheFiles([ + new Query('resource', Query::TYPE_EQUAL, [$this->args['resource']]) + ]); + } + + protected function deleteCacheByTimestamp(): void + { + $this->deleteCacheFiles([ + new Query('accessedAt', Query::TYPE_LESSER, [$this->args['timestamp']]) + ]); + } + + protected function deleteCacheFiles($query): void + { + $this->deleteForProjectIds(function (string $projectId) use ($query) { + + $dbForProject = $this->getProjectDB($projectId); + $cache = new Cache( + new Filesystem(APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $projectId) + ); + + $this->deleteByGroup( + 'cache', + $query, + $dbForProject, + function (Document $document) use ($cache, $projectId) { + $path = APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $projectId . DIRECTORY_SEPARATOR . $document->getId(); + + if ($cache->purge($document->getId())) { + Console::success('Deleting cache file: ' . $path); + } else { + Console::error('Failed to delete cache file: ' . $path); + } + } + ); + }); + } + /** * @param Document $document database document * @param string $projectId @@ -276,10 +327,10 @@ class DeletesV1 extends Worker { $this->deleteForProjectIds(function (string $projectId) use ($timestamp) { $dbForProject = $this->getProjectDB($projectId); - // Delete Sessions - $this->deleteByGroup('sessions', [ + // Delete Sessions + $this->deleteByGroup('sessions', [ new Query('expire', Query::TYPE_LESSER, [$timestamp]) - ], $dbForProject); + ], $dbForProject); }); } diff --git a/docker-compose.yml b/docker-compose.yml index 2392eb49f..3a3c65651 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -103,11 +103,9 @@ services: - ./phpunit.xml:/usr/src/code/phpunit.xml - ./tests:/usr/src/code/tests - ./app:/usr/src/code/app - # - ./vendor/utopia/database:/usr/src/code/vendor/utopia/database - ./docs:/usr/src/code/docs - ./public:/usr/src/code/public - ./src:/usr/src/code/src - # - ./debug:/tmp - ./dev:/usr/local/dev depends_on: - mariadb @@ -173,6 +171,7 @@ services: - _APP_STATSD_PORT - _APP_MAINTENANCE_INTERVAL - _APP_MAINTENANCE_RETENTION_EXECUTION + - _APP_MAINTENANCE_RETENTION_CACHE - _APP_MAINTENANCE_RETENTION_ABUSE - _APP_MAINTENANCE_RETENTION_AUDIT - _APP_SMS_PROVIDER @@ -207,7 +206,6 @@ services: volumes: - ./app:/usr/src/code/app - ./src:/usr/src/code/src - # - ./vendor:/usr/src/code/vendor depends_on: - mariadb - redis @@ -330,7 +328,7 @@ services: volumes: - ./app:/usr/src/code/app - ./src:/usr/src/code/src - # - ./vendor/utopia-php/database:/usr/src/code/vendor/utopia-php/database + #- ./vendor/utopia-php/database:/usr/src/code/vendor/utopia-php/database depends_on: - redis - mariadb @@ -579,6 +577,7 @@ services: - _APP_DB_PASS - _APP_MAINTENANCE_INTERVAL - _APP_MAINTENANCE_RETENTION_EXECUTION + - _APP_MAINTENANCE_RETENTION_CACHE - _APP_MAINTENANCE_RETENTION_ABUSE - _APP_MAINTENANCE_RETENTION_AUDIT diff --git a/phpunit.xml b/phpunit.xml index 537985271..9494f2915 100644 --- a/phpunit.xml +++ b/phpunit.xml @@ -7,7 +7,7 @@ convertWarningsToExceptions="true" processIsolation="false" stopOnFailure="false" - > +> @@ -35,4 +35,4 @@ ./tests/e2e/Services/Functions/FunctionsCustomClientTest.php - + \ No newline at end of file diff --git a/src/Appwrite/Event/Delete.php b/src/Appwrite/Event/Delete.php index 057abe17f..09cccd799 100644 --- a/src/Appwrite/Event/Delete.php +++ b/src/Appwrite/Event/Delete.php @@ -12,6 +12,7 @@ class Delete extends Event protected ?int $timestamp1d = null; protected ?int $timestamp30m = null; protected ?Document $document = null; + protected ?string $resource = null; public function __construct() { @@ -93,6 +94,29 @@ class Delete extends Event return $this; } + /** + * Returns the resource for the delete event. + * + * @return string + */ + public function getResource(): string + { + return $this->resource; + } + + /** + * Sets the resource for the delete event. + * + * @param string $resource + * @return self + */ + public function setResource(string $resource): self + { + $this->resource = $resource; + + return $this; + } + /** * Returns the set document for the delete event. * @@ -103,6 +127,7 @@ class Delete extends Event return $this->document; } + /** * Executes this event and sends it to the deletes worker. * @@ -117,7 +142,8 @@ class Delete extends Event 'document' => $this->document, 'timestamp' => $this->timestamp, 'timestamp1d' => $this->timestamp1d, - 'timestamp30m' => $this->timestamp30m + 'timestamp30m' => $this->timestamp30m, + 'resource' => $this->resource, ]); } } diff --git a/src/Appwrite/Utopia/Response.php b/src/Appwrite/Utopia/Response.php index 3de2f8300..efe60e415 100644 --- a/src/Appwrite/Utopia/Response.php +++ b/src/Appwrite/Utopia/Response.php @@ -478,6 +478,24 @@ class Response extends SwooleResponse return $this->payload; } + /** + * Output response + * + * Generate HTTP response output including the response header (+cookies) and body and prints them. + * + * @param string $body + * + * @return void + */ + public function file(string $body = ''): void + { + $this->payload = [ + 'payload' => $body + ]; + + $this->send($body); + } + /** * YAML * @@ -509,7 +527,6 @@ class Response extends SwooleResponse return $this->payload; } - /** * Function to set a response filter * diff --git a/tests/e2e/Services/Storage/StorageBase.php b/tests/e2e/Services/Storage/StorageBase.php index edcaa0f8a..a0264ec7c 100644 --- a/tests/e2e/Services/Storage/StorageBase.php +++ b/tests/e2e/Services/Storage/StorageBase.php @@ -490,7 +490,7 @@ trait StorageBase $this->assertEquals(204, $file['headers']['status-code']); $this->assertEmpty($file['body']); - + sleep(1); //upload again using the same ID $file = $this->client->call(Client::METHOD_POST, '/storage/buckets/' . $bucketId . '/files', array_merge([ 'content-type' => 'multipart/form-data', diff --git a/tests/e2e/Services/Webhooks/WebhooksCustomServerTest.php b/tests/e2e/Services/Webhooks/WebhooksCustomServerTest.php index 22f58de93..fd34c8bcb 100644 --- a/tests/e2e/Services/Webhooks/WebhooksCustomServerTest.php +++ b/tests/e2e/Services/Webhooks/WebhooksCustomServerTest.php @@ -289,7 +289,6 @@ class WebhooksCustomServerTest extends Scope $this->assertEquals($webhook['headers']['X-Appwrite-Webhook-Id'] ?? '', $this->getProject()['webhookId']); $this->assertEquals($webhook['headers']['X-Appwrite-Webhook-Project-Id'] ?? '', $this->getProject()['$id']); $this->assertEquals(empty($webhook['headers']['X-Appwrite-Webhook-User-Id'] ?? ''), ('server' === $this->getSide())); - $this->assertEquals($webhook['data']['a'], 'b'); return $data;