CodeMirror/appwrite
1
0
Fork 0
mirror of synced 2024-05-29 17:09:48 +12:00
Code Issues Projects Releases Wiki Activity

Merge pull request #766 from christyjacob4/feat-na-maintenance-task

Browse source 
feat: starter for maintenance task
This commit is contained in:
Eldad A. Fux 2020-12-30 15:42:04 +02:00 committed by GitHub
parent 200658cadf e28c4feb97
commit de70f71114
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
20 changed files with 345 additions and 59 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.env
View file

@ -30,3 +30,4 @@ _APP_FUNCTIONS_CONTAINERS=10
_APP_FUNCTIONS_CPUS=1
_APP_FUNCTIONS_MEMORY=128
_APP_FUNCTIONS_MEMORY_SWAP=128
_APP_MAINTENANCE_INTERVAL=86400

5
Dockerfile
View file

@ -96,7 +96,9 @@ ENV _APP_SERVER=swoole \
_APP_FUNCTIONS_MEMORY=128 \
_APP_FUNCTIONS_MEMORY_SWAP=128 \
_APP_SETUP=self-hosted \
_APP_VERSION=$VERSION
_APP_VERSION=$VERSION \
# 1 Day = 86400 s
_APP_MAINTENANCE_INTERVAL=86400
#ENV _APP_SMTP_SECURE ''
#ENV _APP_SMTP_USERNAME ''
#ENV _APP_SMTP_PASSWORD ''
@ -158,6 +160,7 @@ RUN mkdir -p /storage/uploads && \
# Executables
RUN chmod +x /usr/local/bin/doctor && \
chmod +x /usr/local/bin/maintenance && \
chmod +x /usr/local/bin/install && \
chmod +x /usr/local/bin/migrate && \
chmod +x /usr/local/bin/schedule && \

1
app/cli.php
View file

@ -9,6 +9,7 @@ use Utopia\CLI\Console;
$cli = new CLI();
include 'tasks/doctor.php';
include 'tasks/maintenance.php';
include 'tasks/install.php';
include 'tasks/migrate.php';
include 'tasks/sdks.php';

6
app/config/variables.php
View file

@ -169,4 +169,10 @@ return [
'required' => false,
'question' => '',
],
[
'name' => '_APP_MAINTENANCE_INTERVAL',
'default' => '86400',
'required' => false,
'question' => '',
],
];

1
app/controllers/api/database.php
View file

@ -265,6 +265,7 @@ App::delete('/v1/database/collections/:collectionId')
}
$deletes
->setParam('type', DELETE_TYPE_DOCUMENT)
->setParam('document', $collection)
;

1
app/controllers/api/functions.php
View file

@ -380,6 +380,7 @@ App::delete('/v1/functions/:functionId')
}
$deletes
->setParam('type', DELETE_TYPE_DOCUMENT)
->setParam('document', $function->getArrayCopy())
;

25
app/controllers/api/health.php
View file

@ -5,6 +5,7 @@ use Utopia\Exception;
use Appwrite\Storage\Device\Local;
use Appwrite\Storage\Storage;
use Appwrite\ClamAV\Network;
use Appwrite\Event\Event;
App::get('/v1/health')
->desc('Get HTTP')
@ -130,8 +131,8 @@ App::get('/v1/health/queue/webhooks')
->action(function ($response) {
/** @var Appwrite\Utopia\Response $response */
$response->json(['size' => Resque::size('v1-webhooks')]);
});
$response->json(['size' => Resque::size(Event::WEBHOOK_QUEUE_NAME)]);
}, ['response']);
App::get('/v1/health/queue/tasks')
->desc('Get Tasks Queue')
@ -145,8 +146,8 @@ App::get('/v1/health/queue/tasks')
->action(function ($response) {
/** @var Appwrite\Utopia\Response $response */
$response->json(['size' => Resque::size('v1-tasks')]);
});
$response->json(['size' => Resque::size(Event::TASK_QUEUE_NAME)]);
}, ['response']);
App::get('/v1/health/queue/logs')
->desc('Get Logs Queue')
@ -160,8 +161,8 @@ App::get('/v1/health/queue/logs')
->action(function ($response) {
/** @var Appwrite\Utopia\Response $response */
$response->json(['size' => Resque::size('v1-audit')]);
});
$response->json(['size' => Resque::size(Event::AUDITS_QUEUE_NAME)]);
}, ['response']);
App::get('/v1/health/queue/usage')
->desc('Get Usage Queue')
@ -175,8 +176,8 @@ App::get('/v1/health/queue/usage')
->action(function ($response) {
/** @var Appwrite\Utopia\Response $response */
$response->json(['size' => Resque::size('v1-usage')]);
});
$response->json(['size' => Resque::size(Event::USAGE_QUEUE_NAME)]);
}, ['response']);
App::get('/v1/health/queue/certificates')
->desc('Get Certificate Queue')
@ -190,8 +191,8 @@ App::get('/v1/health/queue/certificates')
->action(function ($response) {
/** @var Appwrite\Utopia\Response $response */
$response->json(['size' => Resque::size('v1-certificates')]);
});
$response->json(['size' => Resque::size(Event::CERTIFICATES_QUEUE_NAME)]);
}, ['response']);
App::get('/v1/health/queue/functions')
->desc('Get Functions Queue')
@ -205,8 +206,8 @@ App::get('/v1/health/queue/functions')
->action(function ($response) {
/** @var Appwrite\Utopia\Response $response */
$response->json(['size' => Resque::size('v1-functions')]);
});
$response->json(['size' => Resque::size(Event::FUNCTIONS_QUEUE_NAME)]);
}, ['response']);
App::get('/v1/health/storage/local')
->desc('Get Local Storage')

5
app/controllers/api/projects.php
View file

@ -464,7 +464,10 @@ App::delete('/v1/projects/:projectId')
throw new Exception('Project not found', 404);
}
$deletes->setParam('document', $project->getArrayCopy());
$deletes
->setParam('type', DELETE_TYPE_DOCUMENT)
->setParam('document', $project->getArrayCopy())
;
foreach (['keys', 'webhooks', 'tasks', 'platforms', 'domains'] as $key) { // Delete all children (keys, webhooks, tasks [stop tasks?], platforms)
$list = $project->getAttribute('webhooks', []);

1
app/controllers/api/users.php
View file

@ -545,6 +545,7 @@ App::delete('/v1/users/:userId')
}
$deletes
->setParam('type', DELETE_TYPE_DOCUMENT)
->setParam('document', $user)
;

2
app/controllers/general.php
View file

@ -294,7 +294,7 @@ App::shutdown(function ($utopia, $request, $response, $project, $events, $audits
$audits->trigger();
}
if (!empty($deletes->getParam('document'))) {
if (!empty($deletes->getParam('type')) && !empty($deletes->getParam('document'))) {
$deletes->trigger();
}

15
app/init.php
View file

@ -52,7 +52,12 @@ const APP_SOCIAL_INSTAGRAM = 'https://www.instagram.com/appwrite.io';
const APP_SOCIAL_GITHUB = 'https://github.com/appwrite';
const APP_SOCIAL_DISCORD = 'https://appwrite.io/discord';
const APP_SOCIAL_DEV = 'https://dev.to/appwrite';
const APP_SOCIAL_STACKSHARE = 'https://stackshare.io/appwrite';
const APP_SOCIAL_STACKSHARE = 'https://stackshare.io/appwrite';
// Deletion Types
const DELETE_TYPE_DOCUMENT = 'document';
const DELETE_TYPE_EXECUTIONS = 'executions';
const DELETE_TYPE_AUDIT = 'audit';
const DELETE_TYPE_ABUSE = 'abuse';
$register = new Registry();
@ -298,19 +303,19 @@ App::setResource('events', function($register) {
}, ['register']);
App::setResource('audits', function($register) {
return new Event('v1-audits', 'AuditsV1');
return new Event(Event::AUDITS_QUEUE_NAME, Event::AUDITS_CLASS_NAME);
}, ['register']);
App::setResource('usage', function($register) {
return new Event('v1-usage', 'UsageV1');
return new Event(Event::USAGE_QUEUE_NAME, Event::USAGE_CLASS_NAME);
}, ['register']);
App::setResource('mails', function($register) {
return new Event('v1-mails', 'MailsV1');
return new Event(Event::MAILS_QUEUE_NAME, Event::MAILS_CLASS_NAME);
}, ['register']);
App::setResource('deletes', function($register) {
return new Event('v1-deletes', 'DeletesV1');
return new Event(Event::DELETE_QUEUE_NAME, Event::DELETE_CLASS_NAME);
}, ['register']);
// Test Mock

67
app/tasks/maintenance.php Normal file
View file

@ -0,0 +1,67 @@
<?php
global $cli;
require_once __DIR__.'/../init.php';
use Appwrite\Database\Database;
use Appwrite\Database\Adapter\MySQL as MySQLAdapter;
use Appwrite\Database\Adapter\Redis as RedisAdapter;
use Appwrite\Event\Event;
use Utopia\App;
use Utopia\CLI\Console;
use Utopia\Config\Config;
// TODO: Think of a better way to access consoleDB
function getConsoleDB() {
global $register;
$consoleDB = new Database();
$consoleDB->setAdapter(new RedisAdapter(new MySQLAdapter($register), $register));
$consoleDB->setNamespace('app_console'); // Main DB
$consoleDB->setMocks(Config::getParam('collections', []));
return $consoleDB;
}
function notifyDeleteExecutionLogs()
{
Resque::enqueue(Event::DELETE_QUEUE_NAME, Event::DELETE_CLASS_NAME, [
'type' => DELETE_TYPE_EXECUTIONS
]);
}
function notifyDeleteAbuseLogs(int $interval)
{
Resque::enqueue(Event::DELETE_QUEUE_NAME, Event::DELETE_CLASS_NAME, [
'type' => DELETE_TYPE_ABUSE,
'timestamp' => time() - $interval
]);
}
function notifyDeleteAuditLogs(int $interval)
{
Resque::enqueue(Event::DELETE_QUEUE_NAME, Event::DELETE_CLASS_NAME, [
'type' => DELETE_TYPE_AUDIT,
'timestamp' => time() - $interval
]);
}
$cli
->task('maintenance')
->desc('Schedules maintenance tasks and publishes them to resque')
->action(function () {
// # of days in seconds (1 day = 86400s)
$interval = (int) App::getEnv('_APP_MAINTENANCE_INTERVAL', '86400');
//Convert Seconds to microseconds
$intervalMicroseconds = $interval * 1000000;
$consoleDB = getConsoleDB();
Console::loop(function() use ($consoleDB, $interval){
Console::info("[ MAINTENANCE TASK ] Notifying deletes workers every {$interval} seconds");
notifyDeleteExecutionLogs();
notifyDeleteAbuseLogs($interval);
notifyDeleteAuditLogs($interval);
}, $intervalMicroseconds);
});

21
app/views/install/compose.phtml
View file

@ -261,6 +261,27 @@ services:
- _APP_SMTP_SECURE
- _APP_SMTP_USERNAME
- _APP_SMTP_PASSWORD
appwrite-maintenance:
entrypoint: maintenance
container_name: appwrite-maintenance
build:
context: .
restart: unless-stopped
networks:
- appwrite
depends_on:
- redis
environment:
- _APP_ENV
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_MAINTENANCE_INTERVAL
- _APP_DB_HOST
- _APP_DB_PORT
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
appwrite-schedule:
image: appwrite/appwrite:<?php echo $version."\n"; ?>

170
app/workers/deletes.php
View file

@ -1,22 +1,27 @@
<?php
require_once __DIR__.'/../init.php';
\cli_set_process_title('Deletes V1 Worker');
echo APP_NAME.' deletes worker v1 has started'."\n";
use Appwrite\Database\Database;
use Appwrite\Database\Adapter\MySQL as MySQLAdapter;
use Appwrite\Database\Adapter\Redis as RedisAdapter;
use Appwrite\Database\Document;
use Appwrite\Database\Validator\Authorization;
use Appwrite\Storage\Device\Local;
use Utopia\Abuse\Abuse;
use Utopia\Abuse\Adapters\TimeLimit;
use Utopia\CLI\Console;
use Utopia\Config\Config;
use Utopia\Audit\Audit;
use Utopia\Audit\Adapters\MySQL as AuditAdapter;
require_once __DIR__.'/../init.php';
\cli_set_process_title('Deletes V1 Worker');
Console::success(APP_NAME.' deletes worker v1 has started'."\n");
class DeletesV1
{
public $args = [];
protected $consoleDB = null;
@ -27,28 +32,49 @@ class DeletesV1
public function perform()
{
$projectId = $this->args['projectId'];
$document = $this->args['document'];
$document = new Document($document);
$projectId = $this->args['projectId'];
$type = $this->args['type'];
switch (strval($document->getCollection())) {
case Database::SYSTEM_COLLECTION_PROJECTS:
$this->deleteProject($document);
switch (strval($type)) {
case DELETE_TYPE_DOCUMENT:
$document = $this->args['document'];
$document = new Document($document);
switch (strval($document->getCollection())) {
case Database::SYSTEM_COLLECTION_PROJECTS:
$this->deleteProject($document);
break;
case Database::SYSTEM_COLLECTION_FUNCTIONS:
$this->deleteFunction($document, $projectId);
break;
case Database::SYSTEM_COLLECTION_USERS:
$this->deleteUser($document, $projectId);
break;
case Database::SYSTEM_COLLECTION_COLLECTIONS:
$this->deleteDocuments($document, $projectId);
break;
default:
Console::error('No lazy delete operation available for document of type: '.$document->getCollection());
break;
}
break;
case Database::SYSTEM_COLLECTION_FUNCTIONS:
$this->deleteFunction($document, $projectId);
case DELETE_TYPE_EXECUTIONS:
$this->deleteExecutionLogs();
break;
case Database::SYSTEM_COLLECTION_USERS:
$this->deleteUser($document, $projectId);
case DELETE_TYPE_AUDIT:
$this->deleteAuditLogs($this->args['timestamp']);
break;
case Database::SYSTEM_COLLECTION_COLLECTIONS:
$this->deleteDocuments($document, $projectId);
case DELETE_TYPE_ABUSE:
$this->deleteAbuseLogs($this->args['timestamp']);
break;
default:
Console::error('No lazy delete operation available for document of type: '.$document->getCollection());
Console::error('No delete operation for type: '.$type);
break;
}
}
}
public function tearDown(): void
@ -84,7 +110,7 @@ class DeletesV1
foreach ($tokens as $token) {
if (!$this->getProjectDB($projectId)->deleteDocument($token->getId())) {
throw new Exception('Failed to remove token from DB', 500);
throw new Exception('Failed to remove token from DB');
}
}
@ -95,6 +121,59 @@ class DeletesV1
], $this->getProjectDB($projectId));
}
protected function deleteExecutionLogs()
{
$this->deleteForProjectIds(function($projectId) {
if (!($projectDB = $this->getProjectDB($projectId))) {
throw new Exception('Failed to get projectDB for project '.$projectId);
}
// Delete Executions
$this->deleteByGroup([
'$collection='.Database::SYSTEM_COLLECTION_EXECUTIONS
], $projectDB);
});
}
protected function deleteAbuseLogs($timestamp)
{
global $register;
if($timestamp == 0) {
throw new Exception('Failed to delete audit logs. No timestamp provided');
}
$timeLimit = new TimeLimit("", 0, 1, function () use ($register) {
return $register->get('db');
});
$this->deleteForProjectIds(function($projectId) use ($timeLimit, $timestamp){
$timeLimit->setNamespace('app_'.$projectId);
$abuse = new Abuse($timeLimit);
$status = $abuse->cleanup($timestamp);
if (!$status) {
throw new Exception('Failed to delete Abuse logs for project '.$projectId);
}
});
}
protected function deleteAuditLogs($timestamp)
{
global $register;
if($timestamp == 0) {
throw new Exception('Failed to delete audit logs. No timestamp provided');
}
$this->deleteForProjectIds(function($projectId) use ($register, $timestamp){
$adapter = new AuditAdapter($register->get('db'));
$adapter->setNamespace('app_'.$projectId);
$audit = new Audit($adapter);
$status = $audit->cleanup($timestamp);
if (!$status) {
throw new Exception('Failed to delete Audit logs for project'.$projectId);
}
});
}
protected function deleteFunction(Document $document, $projectId)
{
$projectDB = $this->getProjectDB($projectId);
@ -142,6 +221,48 @@ class DeletesV1
Authorization::reset();
}
protected function deleteForProjectIds(callable $callback)
{
$count = 0;
$chunk = 0;
$limit = 50;
$projects = [];
$sum = $limit;
$executionStart = \microtime(true);
while($sum === $limit) {
$chunk++;
Authorization::disable();
$projects = $this->getConsoleDB()->getCollection([
'limit' => $limit,
'offset' => $count,
'orderType' => 'ASC',
'orderCast' => 'string',
'filters' => [
'$collection='.Database::SYSTEM_COLLECTION_PROJECTS,
],
]);
Authorization::reset();
$projectIds = array_map (function ($project) {
return $project->getId();
}, $projects);
$sum = count($projects);
Console::info('Executing delete function for chunk #'.$chunk.'. Found '.$sum.' projects');
foreach ($projectIds as $projectId) {
$callback($projectId);
$count++;
}
}
$executionEnd = \microtime(true);
Console::info("Found {$count} projects " . ($executionEnd - $executionStart) . " seconds");
}
protected function deleteByGroup(array $filters, Database $database, callable $callback = null)
{
$count = 0;
@ -159,7 +280,10 @@ class DeletesV1
$results = $database->getCollection([
'limit' => $limit,
'offset' => 0,
'offset' => $count,
'orderField' => '$id',
'orderType' => 'ASC',
'orderCast' => 'string',
'filters' => $filters,
]);

2
app/workers/mails.php
View file

@ -7,7 +7,7 @@ require_once __DIR__.'/../init.php';
\cli_set_process_title('Mails V1 Worker');
echo APP_NAME.' mails worker v1 has started'."\n";
Console::success(APP_NAME.' mails worker v1 has started'."\n");
class MailsV1
{

3
bin/maintenance Normal file
View file

@ -0,0 +1,3 @@
#!/bin/sh
php /usr/src/code/app/cli.php maintenance $@

6
composer.json
View file

@ -35,10 +35,10 @@
"appwrite/php-clamav": "1.0.*",
"utopia-php/framework": "0.10.0",
"utopia-php/abuse": "0.2.*",
"utopia-php/audit": "0.3.*",
"utopia-php/abuse": "0.3.*",
"utopia-php/audit": "0.5.*",
"utopia-php/cache": "0.2.*",
"utopia-php/cli": "0.7.3",
"utopia-php/cli": "0.8.0",
"utopia-php/config": "0.2.*",
"utopia-php/locale": "0.3.*",
"utopia-php/registry": "0.2.*",

25
composer.lock generated
View file

@ -1112,16 +1112,16 @@
},
{
"name": "utopia-php/abuse",
"version": "0.2.2",
"version": "0.3.1",
"source": {
"type": "git",
"url": "https://github.com/utopia-php/abuse.git",
"reference": "8e65890a6d7afa9f57992f1eca9fe64508f9822e"
"reference": "23c2eb533bca8f3ef5548ae265398fa7d4d39a1c"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/utopia-php/abuse/zipball/8e65890a6d7afa9f57992f1eca9fe64508f9822e",
"reference": "8e65890a6d7afa9f57992f1eca9fe64508f9822e",
"url": "https://api.github.com/repos/utopia-php/abuse/zipball/23c2eb533bca8f3ef5548ae265398fa7d4d39a1c",
"reference": "23c2eb533bca8f3ef5548ae265398fa7d4d39a1c",
"shasum": ""
},
"require": {
@ -1164,16 +1164,16 @@
},
{
"name": "utopia-php/audit",
"version": "0.3.2",
"version": "0.5.1",
"source": {
"type": "git",
"url": "https://github.com/utopia-php/audit.git",
"reference": "544ecff78788d11f60992a721f102cafc22ab084"
"reference": "154a850170a58667a15e4b65fbabb6cd0b709dd9"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/utopia-php/audit/zipball/544ecff78788d11f60992a721f102cafc22ab084",
"reference": "544ecff78788d11f60992a721f102cafc22ab084",
"url": "https://api.github.com/repos/utopia-php/audit/zipball/154a850170a58667a15e4b65fbabb6cd0b709dd9",
"reference": "154a850170a58667a15e4b65fbabb6cd0b709dd9",
"shasum": ""
},
"require": {
@ -1268,16 +1268,16 @@
},
{
"name": "utopia-php/cli",
"version": "0.7.3",
"version": "0.8",
"source": {
"type": "git",
"url": "https://github.com/utopia-php/cli.git",
"reference": "0337918242278e0cf98f8dcab2e75b5a3153b856"
"reference": "090c7ae22b53a175d962e8f9601d36350496153c"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/utopia-php/cli/zipball/0337918242278e0cf98f8dcab2e75b5a3153b856",
"reference": "0337918242278e0cf98f8dcab2e75b5a3153b856",
"url": "https://api.github.com/repos/utopia-php/cli/zipball/090c7ae22b53a175d962e8f9601d36350496153c",
"reference": "090c7ae22b53a175d962e8f9601d36350496153c",
"shasum": ""
},
"require": {
@ -1875,6 +1875,7 @@
"require-dev": {
"phpunit/phpunit": "^7.0"
},
"default-branch": true,
"type": "library",
"autoload": {
"psr-4": {

21
docker-compose.yml
View file

@ -314,6 +314,27 @@ services:
- _APP_SMTP_USERNAME
- _APP_SMTP_PASSWORD
appwrite-maintenance:
entrypoint: maintenance
container_name: appwrite-maintenance
build:
context: .
restart: unless-stopped
networks:
- appwrite
depends_on:
- redis
environment:
- _APP_ENV
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_MAINTENANCE_INTERVAL
- _APP_DB_HOST
- _APP_DB_PORT
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
appwrite-schedule:
entrypoint: schedule
container_name: appwrite-schedule

26
src/Appwrite/Event/Event.php
View file

@ -6,6 +6,32 @@ use Resque;
class Event
{
const DELETE_QUEUE_NAME = 'v1-deletes';
const DELETE_CLASS_NAME = 'DeletesV1';
const AUDITS_QUEUE_NAME = 'v1-audits';
const AUDITS_CLASS_NAME = 'AuditsV1';
const USAGE_QUEUE_NAME = 'v1-usage';
const USAGE_CLASS_NAME = 'UsageV1';
const MAILS_QUEUE_NAME = 'v1-mails';
const MAILS_CLASS_NAME = 'MailsV1';
const FUNCTIONS_QUEUE_NAME = 'v1-functions';
const FUNCTIONS_CLASS_NAME = 'FunctionsV1';
const WEBHOOK_QUEUE_NAME = 'v1-webhooks';
const WEBHOOK_CLASS_NAME = 'WebhooksV1';
const TASK_QUEUE_NAME = 'v1-tasks';
const TASK_CLASS_NAME = 'TasksV1';
const CERTIFICATES_QUEUE_NAME = 'v1-certificates';
const CERTIFICATES_CLASS_NAME = 'CertificatesV1';
/**
* @var string
*/