From 41d207e862928ef7449133e31719e316207cc053 Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 13 Jun 2023 11:15:38 +0300 Subject: [PATCH 1/2] cache deletion bug fix --- app/controllers/api/storage.php | 8 ++++- app/workers/deletes.php | 61 ++++++++++++++++++++++----------- 2 files changed, 48 insertions(+), 21 deletions(-) diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index 71979cca55..e008c9b8b4 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -1263,7 +1263,8 @@ App::put('/v1/storage/buckets/:bucketId/files/:fileId') ->inject('user') ->inject('mode') ->inject('events') - ->action(function (string $bucketId, string $fileId, ?array $permissions, Response $response, Database $dbForProject, Document $user, string $mode, Event $events) { + ->inject('deletes') + ->action(function (string $bucketId, string $fileId, ?array $permissions, Response $response, Database $dbForProject, Document $user, string $mode, Event $events, Delete $deletes) { $bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); @@ -1329,6 +1330,11 @@ App::put('/v1/storage/buckets/:bucketId/files/:fileId') $file = Authorization::skip(fn() => $dbForProject->updateDocument('bucket_' . $bucket->getInternalId(), $fileId, $file)); } + $deletes + ->setType(DELETE_TYPE_CACHE_BY_RESOURCE) + ->setResource('file/' . $file->getId()) + ; + $events ->setParam('bucketId', $bucket->getId()) ->setParam('fileId', $file->getId()) diff --git a/app/workers/deletes.php b/app/workers/deletes.php index 5ddc4c674a..b01382e9f2 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -111,10 +111,10 @@ class DeletesV1 extends Worker break; case DELETE_TYPE_CACHE_BY_RESOURCE: - $this->deleteCacheByResource($this->args['resource']); + $this->deleteCacheByResource($project, $this->args['resource']); break; case DELETE_TYPE_CACHE_BY_TIMESTAMP: - $this->deleteCacheByDate(); + $this->deleteCacheByDate($this->args['datetime']); break; case DELETE_TYPE_SCHEDULES: $this->deleteSchedules($this->args['datetime']); @@ -162,25 +162,41 @@ class DeletesV1 extends Worker } /** + * @param Document $project * @param string $resource + * @throws Exception */ - protected function deleteCacheByResource(string $resource): void + protected function deleteCacheByResource(Document $project, string $resource): void { - $this->deleteCacheFiles([ - Query::equal('resource', [$resource]), - ]); + $dbForProject = $this->getProjectDB($project); + $projectId = $project->getId(); + $document = $dbForProject->findOne('cache', [Query::equal('resource', [$resource])]); + $cache = new Cache( + new Filesystem(APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $projectId) + ); + + $this->deleteById( + $document, + $dbForProject, + function ($document) use ($cache, $projectId) { + $path = APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $projectId . DIRECTORY_SEPARATOR . $document->getId(); + + if ($cache->purge($document->getId())) { + Console::success('Deleting cache file: ' . $path); + } else { + Console::error('Failed to delete cache file: ' . $path); + } + } + ); } - protected function deleteCacheByDate(): void + /** + * @param string $datetime + * @throws Exception + */ + protected function deleteCacheByDate(string $datetime): void { - $this->deleteCacheFiles([ - Query::lessThan('accessedAt', $this->args['datetime']), - ]); - } - - protected function deleteCacheFiles($query): void - { - $this->deleteForProjectIds(function (Document $project) use ($query) { + $this->deleteForProjectIds(function (Document $project) use ($datetime) { $projectId = $project->getId(); $dbForProject = $this->getProjectDB($project); @@ -188,6 +204,10 @@ class DeletesV1 extends Worker new Filesystem(APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $projectId) ); + $query = [ + Query::lessThan('accessedAt', $datetime), + ]; + $this->deleteByGroup( 'cache', $query, @@ -205,9 +225,10 @@ class DeletesV1 extends Worker }); } + /** * @param Document $document database document - * @param Document $projectId + * @param Document $project */ protected function deleteDatabase(Document $document, Document $project): void { @@ -621,13 +642,13 @@ class DeletesV1 extends Worker try { while ($sum === $limit) { $chunk++; - + $results = $database->find($collection, \array_merge([Query::limit($limit)], $queries)); - + $sum = count($results); - + Console::info('Deleting chunk #' . $chunk . '. Found ' . $sum . ' documents in collection ' . $database->getNamespace() . '_' . $collection); - + foreach ($results as $document) { $this->deleteById($document, $database, $callback); $count++; From 8645757f28371c60436f26f557f539f9ccdc04d1 Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 13 Jun 2023 13:41:54 +0300 Subject: [PATCH 2/2] cache deletion bug fix --- app/controllers/api/storage.php | 5 ----- app/workers/deletes.php | 31 +++++++++++++++++-------------- 2 files changed, 17 insertions(+), 19 deletions(-) diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index e008c9b8b4..8702429d5f 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -642,11 +642,6 @@ App::post('/v1/storage/buckets/:bucketId/files') ->setContext('bucket', $bucket) ; - $deletes - ->setType(DELETE_TYPE_CACHE_BY_RESOURCE) - ->setResource('file/' . $file->getId()) - ; - $metadata = null; // was causing leaks as it was passed by reference $response diff --git a/app/workers/deletes.php b/app/workers/deletes.php index b01382e9f2..e917830e14 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -171,23 +171,26 @@ class DeletesV1 extends Worker $dbForProject = $this->getProjectDB($project); $projectId = $project->getId(); $document = $dbForProject->findOne('cache', [Query::equal('resource', [$resource])]); - $cache = new Cache( - new Filesystem(APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $projectId) - ); - $this->deleteById( - $document, - $dbForProject, - function ($document) use ($cache, $projectId) { - $path = APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $projectId . DIRECTORY_SEPARATOR . $document->getId(); + if ($document) { + $cache = new Cache( + new Filesystem(APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $projectId) + ); - if ($cache->purge($document->getId())) { - Console::success('Deleting cache file: ' . $path); - } else { - Console::error('Failed to delete cache file: ' . $path); + $this->deleteById( + $document, + $dbForProject, + function ($document) use ($cache, $projectId) { + $path = APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $projectId . DIRECTORY_SEPARATOR . $document->getId(); + + if ($cache->purge($document->getId())) { + Console::success('Deleting cache file: ' . $path); + } else { + Console::error('Failed to delete cache file: ' . $path); + } } - } - ); + ); + } } /**