Merge pull request #4860 from appwrite/fix-deletes-worker-tasks
Fix: expired session, schedule deletes
This commit is contained in:
commit
54c6f9f4e1
2
.env
2
.env
|
@ -24,7 +24,7 @@ _APP_DB_SCHEMA=appwrite
|
|||
_APP_DB_USER=user
|
||||
_APP_DB_PASS=password
|
||||
_APP_DB_ROOT_PASS=rootsecretpassword
|
||||
_APP_CONNECTIONS_MAX=251
|
||||
_APP_CONNECTIONS_MAX=3100
|
||||
_APP_POOL_CLIENTS=14
|
||||
_APP_CONNECTIONS_DB_PROJECT=db_fra1_02=mariadb://user:password@mariadb:3306/appwrite
|
||||
_APP_CONNECTIONS_DB_CONSOLE=db_fra1_01=mariadb://user:password@mariadb:3306/appwrite
|
||||
|
|
|
@ -134,8 +134,7 @@ class DeletesV1 extends Worker
|
|||
*/
|
||||
protected function deleteSchedules(string $datetime): void
|
||||
{
|
||||
|
||||
$this->deleteByGroup(
|
||||
$this->listByGroup(
|
||||
'schedules',
|
||||
[
|
||||
Query::equal('region', [App::getEnv('_APP_REGION', 'default')]),
|
||||
|
@ -145,7 +144,6 @@ class DeletesV1 extends Worker
|
|||
],
|
||||
$this->getConsoleDB(),
|
||||
function (Document $document) {
|
||||
Console::info('Querying schedule for function ' . $document->getAttribute('resourceId'));
|
||||
$project = $this->getConsoleDB()->getDocument('projects', $document->getAttribute('projectId'));
|
||||
|
||||
if ($project->isEmpty()) {
|
||||
|
@ -358,8 +356,15 @@ class DeletesV1 extends Worker
|
|||
|
||||
protected function deleteExpiredSessions(): void
|
||||
{
|
||||
$this->deleteForProjectIds(function (Document $project) use ($datetime) {
|
||||
$consoleDB = $this->getConsoleDB();
|
||||
|
||||
$this->deleteForProjectIds(function (Document $project) use ($consoleDB) {
|
||||
$dbForProject = $this->getProjectDB($project);
|
||||
|
||||
$project = $consoleDB->getDocument('projects', $project->getId());
|
||||
$duration = $project->getAttribute('auths', [])['duration'] ?? Auth::TOKEN_EXPIRATION_LOGIN_LONG;
|
||||
$expired = DateTime::addSeconds(new \DateTime(), -1 * $duration);
|
||||
|
||||
// Delete Sessions
|
||||
$this->deleteByGroup('sessions', [
|
||||
Query::lessThan('$createdAt', $expired)
|
||||
|
@ -569,6 +574,7 @@ class DeletesV1 extends Worker
|
|||
*/
|
||||
protected function deleteForProjectIds(callable $callback): void
|
||||
{
|
||||
// TODO: @Meldiron name of this method no longer matches. It does not delete, and it gives whole document
|
||||
$count = 0;
|
||||
$chunk = 0;
|
||||
$limit = 50;
|
||||
|
@ -632,6 +638,43 @@ class DeletesV1 extends Worker
|
|||
Console::info("Deleted {$count} document by group in " . ($executionEnd - $executionStart) . " seconds");
|
||||
}
|
||||
|
||||
/**
|
||||
* @param string $collection collectionID
|
||||
* @param Query[] $queries
|
||||
* @param Database $database
|
||||
* @param callable $callback
|
||||
*/
|
||||
protected function listByGroup(string $collection, array $queries, Database $database, callable $callback = null): void
|
||||
{
|
||||
$count = 0;
|
||||
$chunk = 0;
|
||||
$limit = 50;
|
||||
$results = [];
|
||||
$sum = $limit;
|
||||
|
||||
$executionStart = \microtime(true);
|
||||
|
||||
while ($sum === $limit) {
|
||||
$chunk++;
|
||||
|
||||
$results = $database->find($collection, \array_merge([Query::limit($limit)], $queries));
|
||||
|
||||
$sum = count($results);
|
||||
|
||||
foreach ($results as $document) {
|
||||
if (is_callable($callback)) {
|
||||
$callback($document);
|
||||
}
|
||||
|
||||
$count++;
|
||||
}
|
||||
}
|
||||
|
||||
$executionEnd = \microtime(true);
|
||||
|
||||
Console::info("Listed {$count} document by group in " . ($executionEnd - $executionStart) . " seconds");
|
||||
}
|
||||
|
||||
/**
|
||||
* @param Document $document certificates document
|
||||
*/
|
||||
|
|
3
bin/patch-create-missing-schedules
Normal file
3
bin/patch-create-missing-schedules
Normal file
|
@ -0,0 +1,3 @@
|
|||
#!/bin/sh
|
||||
|
||||
php /usr/src/code/app/cli.php patch-create-missing-schedules $@
|
|
@ -33,15 +33,6 @@ services:
|
|||
- 8080:80
|
||||
- 443:443
|
||||
- 9500:8080
|
||||
ulimits:
|
||||
nofile:
|
||||
soft: 655350
|
||||
hard: 655350
|
||||
sysctls:
|
||||
- net.core.somaxconn=1024
|
||||
- net.ipv4.tcp_rmem=1024 4096 16384
|
||||
- net.ipv4.tcp_wmem=1024 4096 16384
|
||||
- net.ipv4.ip_local_port_range=1025 65535
|
||||
volumes:
|
||||
- /var/run/docker.sock:/var/run/docker.sock
|
||||
- appwrite-config:/storage/config:ro
|
||||
|
|
|
@ -8,6 +8,7 @@ use Appwrite\Platform\Tasks\Install;
|
|||
use Appwrite\Platform\Tasks\Maintenance;
|
||||
use Appwrite\Platform\Tasks\Migrate;
|
||||
use Appwrite\Platform\Tasks\Schedule;
|
||||
use Appwrite\Platform\Tasks\PatchCreateMissingSchedules;
|
||||
use Appwrite\Platform\Tasks\SDKs;
|
||||
use Appwrite\Platform\Tasks\Specs;
|
||||
use Appwrite\Platform\Tasks\SSL;
|
||||
|
@ -29,6 +30,7 @@ class Tasks extends Service
|
|||
->addAction(Doctor::getName(), new Doctor())
|
||||
->addAction(Install::getName(), new Install())
|
||||
->addAction(Maintenance::getName(), new Maintenance())
|
||||
->addAction(PatchCreateMissingSchedules::getName(), new PatchCreateMissingSchedules())
|
||||
->addAction(Schedule::getName(), new Schedule())
|
||||
->addAction(Migrate::getName(), new Migrate())
|
||||
->addAction(SDKs::getName(), new SDKs())
|
||||
|
|
97
src/Appwrite/Platform/Tasks/PatchCreateMissingSchedules.php
Normal file
97
src/Appwrite/Platform/Tasks/PatchCreateMissingSchedules.php
Normal file
|
@ -0,0 +1,97 @@
|
|||
<?php
|
||||
|
||||
namespace Appwrite\Platform\Tasks;
|
||||
|
||||
use Utopia\Platform\Action;
|
||||
use Utopia\CLI\Console;
|
||||
use Utopia\Database\DateTime;
|
||||
use Utopia\Database\Document;
|
||||
use Utopia\Database\Query;
|
||||
use Utopia\Database\Database;
|
||||
use Utopia\Database\ID;
|
||||
use Utopia\Database\Validator\Authorization;
|
||||
|
||||
class PatchCreateMissingSchedules extends Action
|
||||
{
|
||||
public static function getName(): string
|
||||
{
|
||||
return 'patch-create-missing-schedules';
|
||||
}
|
||||
|
||||
public function __construct()
|
||||
{
|
||||
$this
|
||||
->desc('Ensure every function has a schedule')
|
||||
->inject('dbForConsole')
|
||||
->inject('getProjectDB')
|
||||
->callback(fn (Database $dbForConsole, callable $getProjectDB) => $this->action($dbForConsole, $getProjectDB));
|
||||
}
|
||||
|
||||
/**
|
||||
* Iterate over every function on every project to make sure there is a schedule. If not, recreate the schedule.
|
||||
*/
|
||||
public function action(Database $dbForConsole, callable $getProjectDB): void
|
||||
{
|
||||
Authorization::disable();
|
||||
Authorization::setDefaultStatus(false);
|
||||
|
||||
Console::title('PatchCreateMissingSchedules V1');
|
||||
Console::success(APP_NAME . ' PatchCreateMissingSchedules v1 has started');
|
||||
|
||||
$limit = 100;
|
||||
$projectCursor = null;
|
||||
while (true) {
|
||||
$projectsQueries = [Query::limit($limit)];
|
||||
if ($projectCursor !== null) {
|
||||
$projectsQueries[] = Query::cursorAfter($projectCursor);
|
||||
}
|
||||
$projects = $dbForConsole->find('projects', $projectsQueries);
|
||||
|
||||
if (count($projects) === 0) {
|
||||
break;
|
||||
}
|
||||
|
||||
foreach ($projects as $project) {
|
||||
Console::log("Checking Project " . $project->getAttribute('name') . " (" . $project->getId() . ")");
|
||||
$dbForProject = $getProjectDB($project);
|
||||
$functionCursor = null;
|
||||
|
||||
while (true) {
|
||||
$functionsQueries = [Query::limit($limit)];
|
||||
if ($functionCursor !== null) {
|
||||
$functionsQueries[] = Query::cursorAfter($functionCursor);
|
||||
}
|
||||
$functions = $dbForProject->find('functions', $functionsQueries);
|
||||
if (count($functions) === 0) {
|
||||
break;
|
||||
}
|
||||
|
||||
foreach ($functions as $function) {
|
||||
$scheduleId = $function->getAttribute('scheduleId');
|
||||
$schedule = $dbForConsole->getDocument('schedules', $scheduleId);
|
||||
|
||||
if ($schedule->isEmpty()) {
|
||||
$functionId = $function->getId();
|
||||
$schedule = $dbForConsole->createDocument('schedules', new Document([
|
||||
'$id' => ID::custom($scheduleId),
|
||||
'region' => $project->getAttribute('region', 'default'),
|
||||
'resourceType' => 'function',
|
||||
'resourceId' => $functionId,
|
||||
'resourceUpdatedAt' => DateTime::now(),
|
||||
'projectId' => $project->getId(),
|
||||
'schedule' => $function->getAttribute('schedule'),
|
||||
'active' => !empty($function->getAttribute('schedule')) && !empty($function->getAttribute('deployment')),
|
||||
]));
|
||||
|
||||
Console::success('Recreated schedule for function ' . $functionId);
|
||||
}
|
||||
}
|
||||
|
||||
$functionCursor = $functions[array_key_last($functions)];
|
||||
}
|
||||
}
|
||||
|
||||
$projectCursor = $projects[array_key_last($projects)];
|
||||
}
|
||||
}
|
||||
}
|
Loading…
Reference in a new issue