From 07cc9f8a2625ff1abb0d85f605d515de099b00e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Sun, 30 Apr 2023 09:28:37 +0200 Subject: [PATCH 1/6] Implement card cache busting --- app/controllers/api/account.php | 6 +++++ app/controllers/api/avatars.php | 9 +++----- app/controllers/api/users.php | 4 ++++ app/controllers/shared/api.php | 39 +++++++++++++++++++++++++++++++++ 4 files changed, 52 insertions(+), 6 deletions(-) diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index d5d9539a5f..c3c75dd7ae 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -473,6 +473,8 @@ App::get('/v1/account/sessions/oauth2/:provider/redirect') ->label('docs', false) ->label('usage.metric', 'sessions.{scope}.requests.create') ->label('usage.params', ['provider:{request.provider}']) + ->label('cacheBuster', true) + ->label('cacheBuster.resource', ['card-cloud/{user.$id}', 'card-cloud-back/{user.$id}', 'card-cloud-og/{user.$id}']) ->param('provider', '', new WhiteList(\array_keys(Config::getParam('providers')), true), 'OAuth2 provider.') ->param('code', '', new Text(2048), 'OAuth2 code.') ->param('state', '', new Text(2048), 'OAuth2 state params.', true) @@ -1568,6 +1570,8 @@ App::patch('/v1/account/name') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_ACCOUNT) + ->label('cacheBuster', true) + ->label('cacheBuster.resource', ['card-cloud/{user.$id}', 'card-cloud-back/{user.$id}', 'card-cloud-og/{user.$id}']) ->param('name', '', new Text(128), 'User name. Max length: 128 chars.') ->inject('response') ->inject('user') @@ -1639,6 +1643,8 @@ App::patch('/v1/account/email') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_ACCOUNT) + ->label('cacheBuster', true) + ->label('cacheBuster.resource', ['card-cloud/{user.$id}', 'card-cloud-back/{user.$id}', 'card-cloud-og/{user.$id}']) ->param('email', '', new Email(), 'User email.') ->param('password', '', new Password(), 'User password. Must be at least 8 chars.') ->inject('response') diff --git a/app/controllers/api/avatars.php b/app/controllers/api/avatars.php index 4801d87b7a..daa535e734 100644 --- a/app/controllers/api/avatars.php +++ b/app/controllers/api/avatars.php @@ -558,8 +558,7 @@ App::get('/v1/cards/cloud') ->groups(['api', 'avatars']) ->label('scope', 'avatars.read') ->label('cache', true) - ->label('cache.resourceType', 'cards/cloud') - ->label('cache.resource', 'card/{request.userId}') + ->label('cache.resource', 'card-cloud/{request.userId}') ->label('docs', false) ->label('origin', '*') ->param('userId', '', new UID(), 'User ID.', true) @@ -765,8 +764,7 @@ App::get('/v1/cards/cloud-back') ->groups(['api', 'avatars']) ->label('scope', 'avatars.read') ->label('cache', true) - ->label('cache.resourceType', 'cards/cloud-back') - ->label('cache.resource', 'card-back/{request.userId}') + ->label('cache.resource', 'card-cloud-back/{request.userId}') ->label('docs', false) ->label('origin', '*') ->param('userId', '', new UID(), 'User ID.', true) @@ -843,8 +841,7 @@ App::get('/v1/cards/cloud-og') ->groups(['api', 'avatars']) ->label('scope', 'avatars.read') ->label('cache', true) - ->label('cache.resourceType', 'cards/cloud-og') - ->label('cache.resource', 'card-og/{request.userId}') + ->label('cache.resource', 'card-cloud-og/{request.userId}') ->label('docs', false) ->label('origin', '*') ->param('userId', '', new UID(), 'User ID.', true) diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index dce493b024..81419c296e 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -737,6 +737,8 @@ App::patch('/v1/users/:userId/name') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_USER) + ->label('cacheBuster', true) + ->label('cacheBuster.resource', ['card-cloud/{user.$id}', 'card-cloud-back/{user.$id}', 'card-cloud-og/{user.$id}']) ->param('userId', '', new UID(), 'User ID.') ->param('name', '', new Text(128), 'User name. Max length: 128 chars.') ->inject('response') @@ -820,6 +822,8 @@ App::patch('/v1/users/:userId/email') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_USER) + ->label('cacheBuster', true) + ->label('cacheBuster.resource', ['card-cloud/{user.$id}', 'card-cloud-back/{user.$id}', 'card-cloud-og/{user.$id}']) ->param('userId', '', new UID(), 'User ID.') ->param('email', '', new Email(), 'User email.') ->inject('response') diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index db35cebc65..58b0279b24 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -21,6 +21,7 @@ use Utopia\CLI\Console; use Utopia\Database\Database; use Utopia\Database\DateTime; use Utopia\Database\Document; +use Utopia\Database\Query; use Utopia\Database\Validator\Authorization; $parseLabel = function (string $label, array $responsePayload, array $requestParams, Document $user) { @@ -437,6 +438,43 @@ App::shutdown() $database->trigger(); } + /** + * Cache Buster + */ + $bustCache = $route->getLabel('cacheBuster', false); + if ($bustCache) { + $payload = $response->getPayload(); + + if (!empty($payload)) { + $resources = []; + + $patterns = $route->getLabel('cacheBuster.resource', []); + if (!empty($patterns)) { + foreach ($patterns as $pattern) { + $resources[] = $parseLabel($pattern, $responsePayload, $requestParams, $user); + } + } + + $caches = Authorization::skip(fn () => $dbForProject->find('cache', [ + Query::equal('resource', $resources), + Query::limit(100) + ])); + + foreach ($caches as $cache) { + $key = $cache->getId(); + \var_dump("Pruning " . $key); + + $cache = new Cache( + new Filesystem(APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $project->getId()) + ); + + $cache->purge($key); + + Authorization::skip(fn () => $dbForProject->deleteDocument('cache', $cache->getId())); + } + } + } + /** * Cache label */ @@ -468,6 +506,7 @@ App::shutdown() $cacheLog = $dbForProject->getDocument('cache', $key); $accessedAt = $cacheLog->getAttribute('accessedAt', ''); $now = DateTime::now(); + if ($cacheLog->isEmpty()) { Authorization::skip(fn () => $dbForProject->createDocument('cache', new Document([ '$id' => $key, From 1409ea4dbfca9d86d3888e8c6285b06538a76927 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Sun, 30 Apr 2023 09:33:05 +0200 Subject: [PATCH 2/6] Remove leftover --- app/controllers/shared/api.php | 1 - 1 file changed, 1 deletion(-) diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 58b0279b24..de641328c6 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -462,7 +462,6 @@ App::shutdown() foreach ($caches as $cache) { $key = $cache->getId(); - \var_dump("Pruning " . $key); $cache = new Cache( new Filesystem(APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $project->getId()) From ecbc9aecb3fb807e6ccb7e6a99884db9a6f3d3d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Tue, 2 May 2023 13:21:08 +0200 Subject: [PATCH 3/6] Migrated cache buster to a script --- Dockerfile | 1 + app/controllers/api/account.php | 6 -- app/controllers/shared/api.php | 36 ----------- bin/clear-card-cache | 3 + src/Appwrite/Platform/Services/Tasks.php | 2 + .../Platform/Tasks/ClearCardCache.php | 62 +++++++++++++++++++ 6 files changed, 68 insertions(+), 42 deletions(-) create mode 100644 bin/clear-card-cache create mode 100644 src/Appwrite/Platform/Tasks/ClearCardCache.php diff --git a/Dockerfile b/Dockerfile index 37a65d6054..bf3f4f1634 100755 --- a/Dockerfile +++ b/Dockerfile @@ -118,6 +118,7 @@ RUN mkdir -p /storage/uploads && \ # Executables RUN chmod +x /usr/local/bin/doctor && \ chmod +x /usr/local/bin/patch-delete-schedule-updated-at-attribute && \ + chmod +x /usr/local/bin/clear-card-cache && \ chmod +x /usr/local/bin/maintenance && \ chmod +x /usr/local/bin/volume-sync && \ chmod +x /usr/local/bin/usage && \ diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index c3c75dd7ae..d5d9539a5f 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -473,8 +473,6 @@ App::get('/v1/account/sessions/oauth2/:provider/redirect') ->label('docs', false) ->label('usage.metric', 'sessions.{scope}.requests.create') ->label('usage.params', ['provider:{request.provider}']) - ->label('cacheBuster', true) - ->label('cacheBuster.resource', ['card-cloud/{user.$id}', 'card-cloud-back/{user.$id}', 'card-cloud-og/{user.$id}']) ->param('provider', '', new WhiteList(\array_keys(Config::getParam('providers')), true), 'OAuth2 provider.') ->param('code', '', new Text(2048), 'OAuth2 code.') ->param('state', '', new Text(2048), 'OAuth2 state params.', true) @@ -1570,8 +1568,6 @@ App::patch('/v1/account/name') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_ACCOUNT) - ->label('cacheBuster', true) - ->label('cacheBuster.resource', ['card-cloud/{user.$id}', 'card-cloud-back/{user.$id}', 'card-cloud-og/{user.$id}']) ->param('name', '', new Text(128), 'User name. Max length: 128 chars.') ->inject('response') ->inject('user') @@ -1643,8 +1639,6 @@ App::patch('/v1/account/email') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_ACCOUNT) - ->label('cacheBuster', true) - ->label('cacheBuster.resource', ['card-cloud/{user.$id}', 'card-cloud-back/{user.$id}', 'card-cloud-og/{user.$id}']) ->param('email', '', new Email(), 'User email.') ->param('password', '', new Password(), 'User password. Must be at least 8 chars.') ->inject('response') diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index de641328c6..8a2dde5963 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -438,42 +438,6 @@ App::shutdown() $database->trigger(); } - /** - * Cache Buster - */ - $bustCache = $route->getLabel('cacheBuster', false); - if ($bustCache) { - $payload = $response->getPayload(); - - if (!empty($payload)) { - $resources = []; - - $patterns = $route->getLabel('cacheBuster.resource', []); - if (!empty($patterns)) { - foreach ($patterns as $pattern) { - $resources[] = $parseLabel($pattern, $responsePayload, $requestParams, $user); - } - } - - $caches = Authorization::skip(fn () => $dbForProject->find('cache', [ - Query::equal('resource', $resources), - Query::limit(100) - ])); - - foreach ($caches as $cache) { - $key = $cache->getId(); - - $cache = new Cache( - new Filesystem(APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $project->getId()) - ); - - $cache->purge($key); - - Authorization::skip(fn () => $dbForProject->deleteDocument('cache', $cache->getId())); - } - } - } - /** * Cache label */ diff --git a/bin/clear-card-cache b/bin/clear-card-cache new file mode 100644 index 0000000000..b39bc13651 --- /dev/null +++ b/bin/clear-card-cache @@ -0,0 +1,3 @@ +#!/bin/sh + +php /usr/src/code/app/cli.php clear-card-cache $@ \ No newline at end of file diff --git a/src/Appwrite/Platform/Services/Tasks.php b/src/Appwrite/Platform/Services/Tasks.php index a0c5d3a547..b82dca422f 100644 --- a/src/Appwrite/Platform/Services/Tasks.php +++ b/src/Appwrite/Platform/Services/Tasks.php @@ -14,6 +14,7 @@ use Appwrite\Platform\Tasks\Specs; use Appwrite\Platform\Tasks\SSL; use Appwrite\Platform\Tasks\Hamster; use Appwrite\Platform\Tasks\PatchDeleteScheduleUpdatedAtAttribute; +use Appwrite\Platform\Tasks\ClearCardCache; use Appwrite\Platform\Tasks\Usage; use Appwrite\Platform\Tasks\Vars; use Appwrite\Platform\Tasks\Version; @@ -34,6 +35,7 @@ class Tasks extends Service ->addAction(Install::getName(), new Install()) ->addAction(Maintenance::getName(), new Maintenance()) ->addAction(PatchCreateMissingSchedules::getName(), new PatchCreateMissingSchedules()) + ->addAction(ClearCardCache::getName(), new ClearCardCache()) ->addAction(PatchDeleteScheduleUpdatedAtAttribute::getName(), new PatchDeleteScheduleUpdatedAtAttribute()) ->addAction(Schedule::getName(), new Schedule()) ->addAction(Migrate::getName(), new Migrate()) diff --git a/src/Appwrite/Platform/Tasks/ClearCardCache.php b/src/Appwrite/Platform/Tasks/ClearCardCache.php new file mode 100644 index 0000000000..7bee0c1a98 --- /dev/null +++ b/src/Appwrite/Platform/Tasks/ClearCardCache.php @@ -0,0 +1,62 @@ +desc('Deletes card cache for specific user') + ->param('userId', '', new UID(), 'User UID.', false) + ->inject('dbForConsole') + ->callback(fn (string $userId, Database $dbForConsole) => $this->action($userId, $dbForConsole)); + } + + public function action(string $userId, Database $dbForConsole): void + { + Authorization::disable(); + Authorization::setDefaultStatus(false); + + Console::title('ClearCardCache V1'); + Console::success(APP_NAME . ' ClearCardCache v1 has started'); + $resources = ['card-cloud/' . $userId, 'card-cloud-back/' . $userId, 'card-cloud-og/' . $userId]; + + $caches = Authorization::skip(fn () => $dbForConsole->find('cache', [ + Query::equal('resource', $resources), + Query::limit(100) + ])); + + $count = \count($caches); + Console::info("Going to delete {$count} cache records in 5 seconds..."); + \sleep(5); + + foreach ($caches as $cache) { + $key = $cache->getId(); + + $cacheFolder = new Cache( + new Filesystem(APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-console') + ); + + $cacheFolder->purge($key); + + Authorization::skip(fn () => $dbForConsole->deleteDocument('cache', $cache->getId())); + } + + Console::success(APP_NAME . ' ClearCardCache v1 has finished'); + } +} From b9efa52a92348cda8b210834d363ec2c2e7da879 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Tue, 2 May 2023 13:25:38 +0200 Subject: [PATCH 4/6] Revert changes --- app/controllers/api/avatars.php | 9 ++++++--- src/Appwrite/Platform/Tasks/ClearCardCache.php | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/app/controllers/api/avatars.php b/app/controllers/api/avatars.php index daa535e734..4801d87b7a 100644 --- a/app/controllers/api/avatars.php +++ b/app/controllers/api/avatars.php @@ -558,7 +558,8 @@ App::get('/v1/cards/cloud') ->groups(['api', 'avatars']) ->label('scope', 'avatars.read') ->label('cache', true) - ->label('cache.resource', 'card-cloud/{request.userId}') + ->label('cache.resourceType', 'cards/cloud') + ->label('cache.resource', 'card/{request.userId}') ->label('docs', false) ->label('origin', '*') ->param('userId', '', new UID(), 'User ID.', true) @@ -764,7 +765,8 @@ App::get('/v1/cards/cloud-back') ->groups(['api', 'avatars']) ->label('scope', 'avatars.read') ->label('cache', true) - ->label('cache.resource', 'card-cloud-back/{request.userId}') + ->label('cache.resourceType', 'cards/cloud-back') + ->label('cache.resource', 'card-back/{request.userId}') ->label('docs', false) ->label('origin', '*') ->param('userId', '', new UID(), 'User ID.', true) @@ -841,7 +843,8 @@ App::get('/v1/cards/cloud-og') ->groups(['api', 'avatars']) ->label('scope', 'avatars.read') ->label('cache', true) - ->label('cache.resource', 'card-cloud-og/{request.userId}') + ->label('cache.resourceType', 'cards/cloud-og') + ->label('cache.resource', 'card-og/{request.userId}') ->label('docs', false) ->label('origin', '*') ->param('userId', '', new UID(), 'User ID.', true) diff --git a/src/Appwrite/Platform/Tasks/ClearCardCache.php b/src/Appwrite/Platform/Tasks/ClearCardCache.php index 7bee0c1a98..d7fc1465bf 100644 --- a/src/Appwrite/Platform/Tasks/ClearCardCache.php +++ b/src/Appwrite/Platform/Tasks/ClearCardCache.php @@ -34,7 +34,7 @@ class ClearCardCache extends Action Console::title('ClearCardCache V1'); Console::success(APP_NAME . ' ClearCardCache v1 has started'); - $resources = ['card-cloud/' . $userId, 'card-cloud-back/' . $userId, 'card-cloud-og/' . $userId]; + $resources = ['card/' . $userId, 'card-back/' . $userId, 'card-og/' . $userId]; $caches = Authorization::skip(fn () => $dbForConsole->find('cache', [ Query::equal('resource', $resources), From 56172ac196c5f1362757d5f22f428944acb37f96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Tue, 2 May 2023 13:35:53 +0200 Subject: [PATCH 5/6] Update ClearCardCache.php --- src/Appwrite/Platform/Tasks/ClearCardCache.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Appwrite/Platform/Tasks/ClearCardCache.php b/src/Appwrite/Platform/Tasks/ClearCardCache.php index d7fc1465bf..d3153b995c 100644 --- a/src/Appwrite/Platform/Tasks/ClearCardCache.php +++ b/src/Appwrite/Platform/Tasks/ClearCardCache.php @@ -42,8 +42,8 @@ class ClearCardCache extends Action ])); $count = \count($caches); - Console::info("Going to delete {$count} cache records in 5 seconds..."); - \sleep(5); + Console::info("Going to delete {$count} cache records in 10 seconds..."); + \sleep(10); foreach ($caches as $cache) { $key = $cache->getId(); From 94f2853a5f40270d33c0ef9b9d048fc6a079e6aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Tue, 2 May 2023 13:37:41 +0200 Subject: [PATCH 6/6] Remove leftovers --- app/controllers/api/users.php | 4 ---- app/controllers/shared/api.php | 2 -- 2 files changed, 6 deletions(-) diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index 81419c296e..dce493b024 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -737,8 +737,6 @@ App::patch('/v1/users/:userId/name') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_USER) - ->label('cacheBuster', true) - ->label('cacheBuster.resource', ['card-cloud/{user.$id}', 'card-cloud-back/{user.$id}', 'card-cloud-og/{user.$id}']) ->param('userId', '', new UID(), 'User ID.') ->param('name', '', new Text(128), 'User name. Max length: 128 chars.') ->inject('response') @@ -822,8 +820,6 @@ App::patch('/v1/users/:userId/email') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_USER) - ->label('cacheBuster', true) - ->label('cacheBuster.resource', ['card-cloud/{user.$id}', 'card-cloud-back/{user.$id}', 'card-cloud-og/{user.$id}']) ->param('userId', '', new UID(), 'User ID.') ->param('email', '', new Email(), 'User email.') ->inject('response') diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 8a2dde5963..db35cebc65 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -21,7 +21,6 @@ use Utopia\CLI\Console; use Utopia\Database\Database; use Utopia\Database\DateTime; use Utopia\Database\Document; -use Utopia\Database\Query; use Utopia\Database\Validator\Authorization; $parseLabel = function (string $label, array $responsePayload, array $requestParams, Document $user) { @@ -469,7 +468,6 @@ App::shutdown() $cacheLog = $dbForProject->getDocument('cache', $key); $accessedAt = $cacheLog->getAttribute('accessedAt', ''); $now = DateTime::now(); - if ($cacheLog->isEmpty()) { Authorization::skip(fn () => $dbForProject->createDocument('cache', new Document([ '$id' => $key,