CodeMirror/appwrite
1
0
Fork 0
mirror of synced 2024-06-01 18:39:57 +12:00
Code Issues Projects Releases Wiki Activity

cache cleanup

Browse source
This commit is contained in:
shimon 2022-07-03 12:36:59 +03:00
parent 58bec7b034
commit 7b429bdcd7
9 changed files with 161 additions and 5389 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
.env
View file

@ -71,7 +71,8 @@ _APP_FUNCTIONS_INACTIVE_THRESHOLD=60
OPEN_RUNTIMES_NETWORK=appwrite_runtimes
_APP_EXECUTOR_SECRET=your-secret-key
_APP_EXECUTOR_HOST=http://appwrite-executor/v1
_APP_MAINTENANCE_INTERVAL=86400
_APP_MAINTENANCE_INTERVAL=2592000
_APP_MAINTENANCE_RETENTION_CACHE=2592000
_APP_MAINTENANCE_RETENTION_EXECUTION=1209600
_APP_MAINTENANCE_RETENTION_ABUSE=86400
_APP_MAINTENANCE_RETENTION_AUDIT=1209600

39
app/config/collections.php
View file

@ -2764,6 +2764,45 @@ $collections = [
],
]
],
'cache' => [
'$collection' => Database::METADATA,
'$id' => 'cache',
'name' => 'Cache',
'attributes' => [
[
'$id' => 'dateAccessed',
'type' => Database::VAR_INTEGER,
'format' => '',
'size' => 0,
'signed' => false,
'required' => true,
'default' => null,
'array' => false,
'filters' => [],
],
[
'array' => false,
'$id' => 'path',
'type' => Database::VAR_STRING,
'format' => '',
'size' => 100,
'signed' => true,
'required' => true,
'default' => null,
'filters' => [],
],
],
'indexes' => [
[
'$id' => '_key_accessed',
'type' => Database::INDEX_KEY,
'attributes' => ['dateAccessed'],
'lengths' => [],
'orders' => [],
],
],
],
'files' => [
'$collection' => 'buckets',
'$id' => 'files',

60
app/controllers/api/avatars.php
View file

@ -10,7 +10,9 @@ use Utopia\App;
use Utopia\Cache\Adapter\Filesystem;
use Utopia\Cache\Cache;
use Utopia\Config\Config;
use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\Database\Validator\Authorization;
use Utopia\Image\Image;
use Utopia\Validator\Boolean;
use Utopia\Validator\HexColor;
@ -18,7 +20,7 @@ use Utopia\Validator\Range;
use Utopia\Validator\Text;
use Utopia\Validator\WhiteList;
$avatarCallback = function (string $type, string $code, int $width, int $height, int $quality, Response $response) {
$avatarCallback = function (string $type, string $code, int $width, int $height, int $quality, Response $response, Database $dbForProject) {
$code = \strtolower($code);
$type = \strtolower($type);
@ -48,10 +50,21 @@ $avatarCallback = function (string $type, string $code, int $width, int $height,
$cache = new Cache(new Filesystem(APP_STORAGE_CACHE . '/app-0')); // Limit file number or size
$data = $cache->load($key, 60 * 60 * 24 * 30 * 3/* 3 months */);
if ($data) {
//$output = (empty($output)) ? $type : $output;
$cacheRow = $dbForProject->getDocument('cache', $key);
if($cacheRow->isEmpty()){
Authorization::skip(fn () => $dbForProject->createDocument('cache', new Document([
'$id' => $key,
'dateAccessed' => time(),
'path' => 'app-0'
])));
} else {
$cacheRow->setAttribute('dateAccessed', time());
Authorization::skip(fn () => $dbForProject->updateDocument('cache', $cacheRow->getId(), $cacheRow));
}
return $response
->setContentType('image/png')
->addHeader('Expires', $date)
@ -94,7 +107,8 @@ App::get('/v1/avatars/credit-cards/:code')
->param('height', 100, new Range(0, 2000), 'Image height. Pass an integer between 0 to 2000. Defaults to 100.', true)
->param('quality', 100, new Range(0, 100), 'Image quality. Pass an integer between 0 to 100. Defaults to 100.', true)
->inject('response')
->action(fn (string $code, int $width, int $height, int $quality, Response $response) => $avatarCallback('credit-cards', $code, $width, $height, $quality, $response));
->inject('dbForProject')
->action(fn (string $code, int $width, int $height, int $quality, Response $response, Database $dbForProject) => $avatarCallback('credit-cards', $code, $width, $height, $quality, $response, $dbForProject));
App::get('/v1/avatars/browsers/:code')
->desc('Get Browser Icon')
@ -112,7 +126,8 @@ App::get('/v1/avatars/browsers/:code')
->param('height', 100, new Range(0, 2000), 'Image height. Pass an integer between 0 to 2000. Defaults to 100.', true)
->param('quality', 100, new Range(0, 100), 'Image quality. Pass an integer between 0 to 100. Defaults to 100.', true)
->inject('response')
->action(fn (string $code, int $width, int $height, int $quality, Response $response) => $avatarCallback('browsers', $code, $width, $height, $quality, $response));
->inject('dbForProject')
->action(fn (string $code, int $width, int $height, int $quality, Response $response, Database $dbForProject) => $avatarCallback('browsers', $code, $width, $height, $quality, $response, $dbForProject));
App::get('/v1/avatars/flags/:code')
->desc('Get Country Flag')
@ -130,7 +145,8 @@ App::get('/v1/avatars/flags/:code')
->param('height', 100, new Range(0, 2000), 'Image height. Pass an integer between 0 to 2000. Defaults to 100.', true)
->param('quality', 100, new Range(0, 100), 'Image quality. Pass an integer between 0 to 100. Defaults to 100.', true)
->inject('response')
->action(fn (string $code, int $width, int $height, int $quality, Response $response) => $avatarCallback('flags', $code, $width, $height, $quality, $response));
->inject('dbForProject')
->action(fn (string $code, int $width, int $height, int $quality, Response $response, Database $dbForProject) => $avatarCallback('flags', $code, $width, $height, $quality, $response, $dbForProject));
App::get('/v1/avatars/image')
->desc('Get Image from URL')
@ -147,7 +163,8 @@ App::get('/v1/avatars/image')
->param('width', 400, new Range(0, 2000), 'Resize preview image width, Pass an integer between 0 to 2000. Defaults to 400.', true)
->param('height', 400, new Range(0, 2000), 'Resize preview image height, Pass an integer between 0 to 2000. Defaults to 400.', true)
->inject('response')
->action(function (string $url, int $width, int $height, Response $response) {
->inject('dbForProject')
->action(function (string $url, int $width, int $height, Response $response, Database $dbForProject) {
$quality = 80;
$output = 'png';
@ -158,6 +175,20 @@ App::get('/v1/avatars/image')
$data = $cache->load($key, 60 * 60 * 24 * 7/* 1 week */);
if ($data) {
$cacheRow = $dbForProject->getDocument('cache', $key);
if($cacheRow->isEmpty()){
Authorization::skip(fn () => $dbForProject->createDocument('cache', new Document([
'$id' => $key,
'dateAccessed' => time(),
'path' => 'app-0'
])));
} else {
$cacheRow->setAttribute('dateAccessed', time());
Authorization::skip(fn () => $dbForProject->updateDocument('cache', $cacheRow->getId(), $cacheRow));
}
return $response
->setContentType('image/png')
->addHeader('Expires', $date)
@ -211,7 +242,8 @@ App::get('/v1/avatars/favicon')
->label('sdk.response.type', Response::CONTENT_TYPE_IMAGE)
->param('url', '', new URL(['http', 'https']), 'Website URL which you want to fetch the favicon from.')
->inject('response')
->action(function (string $url, Response $response) {
->inject('dbForProject')
->action(function (string $url, Response $response, Database $dbForProject) {
$width = 56;
$height = 56;
@ -222,8 +254,20 @@ App::get('/v1/avatars/favicon')
$type = 'png';
$cache = new Cache(new Filesystem(APP_STORAGE_CACHE . '/app-0')); // Limit file number or size
$data = $cache->load($key, 60 * 60 * 24 * 30 * 3/* 3 months */);
if ($data) {
$cacheRow = $dbForProject->getDocument('cache', $key);
if($cacheRow->isEmpty()){
Authorization::skip(fn () => $dbForProject->createDocument('cache', new Document([
'$id' => $key,
'dateAccessed' => time(),
'path' => 'app-0'
])));
} else {
$cacheRow->setAttribute('dateAccessed', time());
Authorization::skip(fn () => $dbForProject->updateDocument('cache', $cacheRow->getId(), $cacheRow));
}
return $response
->setContentType('image/png')
->addHeader('Expires', $date)

20
app/controllers/api/storage.php
View file

@ -862,7 +862,7 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview')
$fileLogos = Config::getParam('storage-logos');
$date = \date('D, d M Y H:i:s', \time() + (60 * 60 * 24 * 45)) . ' GMT'; // 45 days cache
$key = \md5($fileId . $width . $height . $gravity . $quality . $borderWidth . $borderColor . $borderRadius . $opacity . $rotation . $background . $output);
$key = \md5('preview' . $fileId . $width . $height . $gravity . $quality . $borderWidth . $borderColor . $borderRadius . $opacity . $rotation . $background . $output);
if ($bucket->getAttribute('permission') === 'bucket') {
// skip authorization
@ -893,7 +893,7 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview')
$cipher = null;
$background = (empty($background)) ? 'eceff1' : $background;
$type = \strtolower(\pathinfo($path, PATHINFO_EXTENSION));
$key = \md5($path . $width . $height . $gravity . $quality . $borderWidth . $borderColor . $borderRadius . $opacity . $rotation . $background . $output);
$key = \md5('preview' . $path . $width . $height . $gravity . $quality . $borderWidth . $borderColor . $borderRadius . $opacity . $rotation . $background . $output);
$deviceFiles = $deviceLocal;
}
@ -904,9 +904,8 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview')
throw new Exception('File not found', 404, Exception::STORAGE_FILE_NOT_FOUND);
}
$cache = new Cache(new Filesystem(APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $project->getId() . DIRECTORY_SEPARATOR . $bucketId . DIRECTORY_SEPARATOR . $fileId)); // Limit file number or size
$cache = new Cache(new Filesystem(APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $project->getId())); // Limit file number or size
$data = $cache->load($key, 60 * 60 * 24 * 30 * 3/* 3 months */);
if (empty($output)) {
// when file extension is not provided and the mime type is not one of our supported outputs
// we fallback to `jpg` output format
@ -914,6 +913,19 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview')
}
if ($data) {
$cacheRow = $dbForProject->getDocument('cache', $key);
if($cacheRow->isEmpty()){
Authorization::skip(fn () => $dbForProject->createDocument('cache', new Document([
'$id' => $key,
'dateAccessed' => time(),
'path' => 'app-' . $project->getId()
])));
} else {
$cacheRow->setAttribute('dateAccessed', time());
Authorization::skip(fn () => $dbForProject->updateDocument('cache', $cacheRow->getId(), $cacheRow));
}
return $response
->setContentType((\array_key_exists($output, $outputs)) ? $outputs[$output] : $outputs['jpg'])
->addHeader('Expires', $date)

1
app/init.php
View file

@ -141,6 +141,7 @@ const DELETE_TYPE_USAGE = 'usage';
const DELETE_TYPE_REALTIME = 'realtime';
const DELETE_TYPE_BUCKETS = 'buckets';
const DELETE_TYPE_SESSIONS = 'sessions';
const DELETE_TYPE_CACHE = 'cache';
// Mail Types
const MAIL_TYPE_VERIFICATION = 'verification';
const MAIL_TYPE_MAGIC_SESSION = 'magicSession';

13
app/tasks/maintenance.php
View file

@ -127,6 +127,15 @@ $cli
}
}
function notifyDeleteCache($interval)
{
(new Delete())
->setType(DELETE_TYPE_CACHE)
->setTimestamp(time() - $interval)
->trigger();
}
// # of days in seconds (1 day = 86400s)
$interval = (int) App::getEnv('_APP_MAINTENANCE_INTERVAL', '86400');
$executionLogsRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_EXECUTION', '1209600');
@ -134,8 +143,9 @@ $cli
$abuseLogsRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_ABUSE', '86400');
$usageStatsRetention30m = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_USAGE_30M', '129600'); //36 hours
$usageStatsRetention1d = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_USAGE_1D', '8640000'); // 100 days
$cacheRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_CACHE', '2592000'); // 30 days
Console::loop(function () use ($interval, $executionLogsRetention, $abuseLogsRetention, $auditLogRetention, $usageStatsRetention30m, $usageStatsRetention1d) {
Console::loop(function () use ($interval, $executionLogsRetention, $abuseLogsRetention, $auditLogRetention, $usageStatsRetention30m, $usageStatsRetention1d, $cacheRetention) {
$database = getConsoleDB();
$time = date('d-m-Y H:i:s', time());
@ -147,5 +157,6 @@ $cli
notifyDeleteConnections();
notifyDeleteExpiredSessions();
renewCertificates($database);
notifyDeleteCache($cacheRetention);
}, $interval);
});

37
app/workers/deletes.php
View file

@ -36,6 +36,7 @@ class DeletesV1 extends Worker
public function run(): void
{
$project = new Document($this->args['project'] ?? []);
$type = $this->args['type'] ?? '';
@ -112,6 +113,11 @@ class DeletesV1 extends Worker
case DELETE_TYPE_USAGE:
$this->deleteUsageStats($this->args['timestamp1d'], $this->args['timestamp30m']);
break;
case DELETE_TYPE_CACHE:
$this->deleteCache($this->args['timestamp']);
break;
default:
Console::error('No delete operation for type: ' . $type);
break;
@ -122,6 +128,31 @@ class DeletesV1 extends Worker
{
}
/**
* @param int $timestamp
*/
protected function deleteCache( int $timestamp): void
{
$this->deleteForProjectIds(function (string $projectId) use ($timestamp) {
$dbForProject = $this->getProjectDB($projectId);
$cache = new Local(APP_STORAGE_CACHE);
$this->deleteByGroup('cache', [
new Query('dateAccessed', Query::TYPE_LESSER, [$timestamp])
], $dbForProject
, function (Document $document) use ($cache) {
$path = $cache->getRoot() . '/' . $document->getAttribute('path') . '/' . $document->getId();
if ($cache->delete($path)) {
Console::success('Deleting cache file: ' . $path);
} else {
Console::error('**Failed to delete cache file: ' . $path);
}
});
});
}
/**
* @param Document $document database document
* @param string $projectId
@ -279,12 +310,16 @@ class DeletesV1 extends Worker
*/
protected function deleteExpiredSessions(int $timestamp): void
{
$this->deleteForProjectIds(function (string $projectId) use ($timestamp) {
$this->deleteForProjectIds(
function (string $projectId) use ($timestamp) {
$dbForProject = $this->getProjectDB($projectId);
// Delete Sessions
$this->deleteByGroup('sessions', [
new Query('expire', Query::TYPE_LESSER, [$timestamp])
], $dbForProject);
});
}

5374
composer.lock generated
View file

File diff suppressed because it is too large Load diff

3
docker-compose.yml
View file

@ -107,6 +107,7 @@ services:
- ./docs:/usr/src/code/docs
- ./public:/usr/src/code/public
- ./src:/usr/src/code/src
- ./vendor:/usr/src/code/vendor
# - ./debug:/tmp
- ./dev:/usr/local/dev
depends_on:
@ -301,6 +302,7 @@ services:
- appwrite-certificates:/storage/certificates:rw
- ./app:/usr/src/code/app
- ./src:/usr/src/code/src
- ./vendor:/usr/src/code/vendor
environment:
- _APP_ENV
- _APP_OPENSSL_KEY_V1
@ -561,6 +563,7 @@ services:
volumes:
- ./app:/usr/src/code/app
- ./src:/usr/src/code/src
- ./vendor:/usr/src/code/vendor
depends_on:
- redis
environment: