From 6279784009cb602eb5f86c3900d7caa7cf5ebe39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Wed, 14 Dec 2022 09:44:40 +0100 Subject: [PATCH 1/5] Fix expired session, schedule deletes --- .env | 4 ++-- app/workers/deletes.php | 51 +++++++++++++++++++++++++++++++++++++---- 2 files changed, 49 insertions(+), 6 deletions(-) diff --git a/.env b/.env index f042b1686d..fee6c6653d 100644 --- a/.env +++ b/.env @@ -24,7 +24,7 @@ _APP_DB_SCHEMA=appwrite _APP_DB_USER=user _APP_DB_PASS=password _APP_DB_ROOT_PASS=rootsecretpassword -_APP_CONNECTIONS_MAX=251 +_APP_CONNECTIONS_MAX=3100 _APP_POOL_CLIENTS=14 _APP_CONNECTIONS_DB_PROJECT=db_fra1_02=mariadb://user:password@mariadb:3306/appwrite _APP_CONNECTIONS_DB_CONSOLE=db_fra1_01=mariadb://user:password@mariadb:3306/appwrite @@ -64,7 +64,7 @@ _APP_MAINTENANCE_RETENTION_CACHE=2592000 _APP_MAINTENANCE_RETENTION_EXECUTION=1209600 _APP_MAINTENANCE_RETENTION_ABUSE=86400 _APP_MAINTENANCE_RETENTION_AUDIT=1209600 -_APP_MAINTENANCE_RETENTION_SCHEDULES=86400 +_APP_MAINTENANCE_RETENTION_SCHEDULES=600 _APP_MAINTENANCE_RETENTION_USAGE_HOURLY=8640000 _APP_USAGE_STATS=enabled _APP_USAGE_AGGREGATION_INTERVAL=30 diff --git a/app/workers/deletes.php b/app/workers/deletes.php index b8efa38ae6..458b341a45 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -134,8 +134,7 @@ class DeletesV1 extends Worker */ protected function deleteSchedules(string $datetime): void { - - $this->deleteByGroup( + $this->listByGroup( 'schedules', [ Query::equal('region', [App::getEnv('_APP_REGION', 'default')]), @@ -145,7 +144,6 @@ class DeletesV1 extends Worker ], $this->getConsoleDB(), function (Document $document) { - Console::info('Querying schedule for function ' . $document->getAttribute('resourceId')); $project = $this->getConsoleDB()->getDocument('projects', $document->getAttribute('projectId')); if ($project->isEmpty()) { @@ -358,8 +356,15 @@ class DeletesV1 extends Worker protected function deleteExpiredSessions(): void { - $this->deleteForProjectIds(function (Document $project) use ($datetime) { + $consoleDB = $this->getConsoleDB(); + + $this->deleteForProjectIds(function (Document $project) use ($consoleDB) { $dbForProject = $this->getProjectDB($project); + + $project = $consoleDB->getDocument('projects', $project->getId()); + $duration = $project->getAttribute('auths', [])['duration'] ?? Auth::TOKEN_EXPIRATION_LOGIN_LONG; + $expired = DateTime::addSeconds(new \DateTime(), -1 * $duration); + // Delete Sessions $this->deleteByGroup('sessions', [ Query::lessThan('$createdAt', $expired) @@ -569,6 +574,7 @@ class DeletesV1 extends Worker */ protected function deleteForProjectIds(callable $callback): void { + // TODO: @Meldiron name of this method no longer matches. It does not delete, and it gives whole document $count = 0; $chunk = 0; $limit = 50; @@ -632,6 +638,43 @@ class DeletesV1 extends Worker Console::info("Deleted {$count} document by group in " . ($executionEnd - $executionStart) . " seconds"); } + /** + * @param string $collection collectionID + * @param Query[] $queries + * @param Database $database + * @param callable $callback + */ + protected function listByGroup(string $collection, array $queries, Database $database, callable $callback = null): void + { + $count = 0; + $chunk = 0; + $limit = 50; + $results = []; + $sum = $limit; + + $executionStart = \microtime(true); + + while ($sum === $limit) { + $chunk++; + + $results = $database->find($collection, \array_merge([Query::limit($limit)], $queries)); + + $sum = count($results); + + foreach ($results as $document) { + if (is_callable($callback)) { + $callback($document); + } + + $count++; + } + } + + $executionEnd = \microtime(true); + + Console::info("Listed {$count} document by group in " . ($executionEnd - $executionStart) . " seconds"); + } + /** * @param Document $document certificates document */ From 41a6a6dbf79273a7143b410b0dc69d78a2cd60a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Wed, 14 Dec 2022 09:51:28 +0100 Subject: [PATCH 2/5] Add patch CLI task --- bin/create-missing-schedules | 3 + src/Appwrite/Platform/Services/Tasks.php | 2 + .../Platform/Tasks/CreateMissingSchedules.php | 97 +++++++++++++++++++ 3 files changed, 102 insertions(+) create mode 100644 bin/create-missing-schedules create mode 100644 src/Appwrite/Platform/Tasks/CreateMissingSchedules.php diff --git a/bin/create-missing-schedules b/bin/create-missing-schedules new file mode 100644 index 0000000000..41750b163c --- /dev/null +++ b/bin/create-missing-schedules @@ -0,0 +1,3 @@ +#!/bin/sh + +php /usr/src/code/app/cli.php create-missing-schedules $@ \ No newline at end of file diff --git a/src/Appwrite/Platform/Services/Tasks.php b/src/Appwrite/Platform/Services/Tasks.php index 2968a66b95..e6919f2cc7 100644 --- a/src/Appwrite/Platform/Services/Tasks.php +++ b/src/Appwrite/Platform/Services/Tasks.php @@ -8,6 +8,7 @@ use Appwrite\Platform\Tasks\Install; use Appwrite\Platform\Tasks\Maintenance; use Appwrite\Platform\Tasks\Migrate; use Appwrite\Platform\Tasks\Schedule; +use Appwrite\Platform\Tasks\CreateMissingSchedules; use Appwrite\Platform\Tasks\SDKs; use Appwrite\Platform\Tasks\Specs; use Appwrite\Platform\Tasks\SSL; @@ -29,6 +30,7 @@ class Tasks extends Service ->addAction(Doctor::getName(), new Doctor()) ->addAction(Install::getName(), new Install()) ->addAction(Maintenance::getName(), new Maintenance()) + ->addAction(CreateMissingSchedules::getName(), new CreateMissingSchedules()) ->addAction(Schedule::getName(), new Schedule()) ->addAction(Migrate::getName(), new Migrate()) ->addAction(SDKs::getName(), new SDKs()) diff --git a/src/Appwrite/Platform/Tasks/CreateMissingSchedules.php b/src/Appwrite/Platform/Tasks/CreateMissingSchedules.php new file mode 100644 index 0000000000..75e1d5b637 --- /dev/null +++ b/src/Appwrite/Platform/Tasks/CreateMissingSchedules.php @@ -0,0 +1,97 @@ +desc('Ensure every function has a schedule') + ->inject('dbForConsole') + ->inject('getProjectDB') + ->callback(fn (Database $dbForConsole, callable $getProjectDB) => $this->action($dbForConsole, $getProjectDB)); + } + + /** + * Iterate over every function on every project to make sure there is a schedule. If not, recreate the schedule. + */ + public function action(Database $dbForConsole, callable $getProjectDB): void + { + Authorization::disable(); + Authorization::setDefaultStatus(false); + + Console::title('CreateMissingSchedules V1'); + Console::success(APP_NAME . ' CreateMissingSchedules v1 has started'); + + $limit = 100; + $projectCursor = null; + while (true) { + $projectsQueries = [Query::limit($limit)]; + if ($projectCursor !== null) { + $projectsQueries[] = Query::cursorAfter($projectCursor); + } + $projects = $dbForConsole->find('projects', $projectsQueries); + + if (count($projects) === 0) { + break; + } + + foreach ($projects as $project) { + Console::log("Checking Project " . $project->getAttribute('name') . " (" . $project->getId() . ")"); + $dbForProject = $getProjectDB($project); + $functionCursor = null; + + while (true) { + $functionsQueries = [Query::limit($limit)]; + if ($functionCursor !== null) { + $functionsQueries[] = Query::cursorAfter($functionCursor); + } + $functions = $dbForProject->find('functions', $functionsQueries); + if (count($functions) === 0) { + break; + } + + foreach ($functions as $function) { + $scheduleId = $function->getAttribute('scheduleId'); + $schedule = $dbForConsole->getDocument('schedules', $scheduleId); + + if ($schedule->isEmpty()) { + $functionId = $function->getId(); + $schedule = $dbForConsole->createDocument('schedules', new Document([ + '$id' => ID::custom($scheduleId), + 'region' => $project->getAttribute('region', 'default'), + 'resourceType' => 'function', + 'resourceId' => $functionId, + 'resourceUpdatedAt' => DateTime::now(), + 'projectId' => $project->getId(), + 'schedule' => $function->getAttribute('schedule'), + 'active' => !empty($function->getAttribute('schedule')) && !empty($function->getAttribute('deployment')), + ])); + + Console::success('Recreated schedule for function ' . $functionId); + } + } + + $functionCursor = $functions[array_key_last($functions)]; + } + } + + $projectCursor = $projects[array_key_last($projects)]; + } + } +} From 11b3abebd1f57f4f32f9f644b6b07d84d28111fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Wed, 14 Dec 2022 09:58:45 +0100 Subject: [PATCH 3/5] Remove tcp setup --- docker-compose.yml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 6c040df4cc..7620039b63 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -33,15 +33,6 @@ services: - 8080:80 - 443:443 - 9500:8080 - ulimits: - nofile: - soft: 655350 - hard: 655350 - sysctls: - - net.core.somaxconn=1024 - - net.ipv4.tcp_rmem=1024 4096 16384 - - net.ipv4.tcp_wmem=1024 4096 16384 - - net.ipv4.ip_local_port_range=1025 65535 volumes: - /var/run/docker.sock:/var/run/docker.sock - appwrite-config:/storage/config:ro From 3d07600a48b1d0d709a571703409221268711f0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Wed, 14 Dec 2022 10:44:46 +0100 Subject: [PATCH 4/5] PR review changes --- bin/create-missing-schedules | 3 --- bin/patch-create-missing-schedules | 3 +++ src/Appwrite/Platform/Services/Tasks.php | 4 ++-- ...ssingSchedules.php => PatchCreateMissingSchedules.php} | 8 ++++---- 4 files changed, 9 insertions(+), 9 deletions(-) delete mode 100644 bin/create-missing-schedules create mode 100644 bin/patch-create-missing-schedules rename src/Appwrite/Platform/Tasks/{CreateMissingSchedules.php => PatchCreateMissingSchedules.php} (93%) diff --git a/bin/create-missing-schedules b/bin/create-missing-schedules deleted file mode 100644 index 41750b163c..0000000000 --- a/bin/create-missing-schedules +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -php /usr/src/code/app/cli.php create-missing-schedules $@ \ No newline at end of file diff --git a/bin/patch-create-missing-schedules b/bin/patch-create-missing-schedules new file mode 100644 index 0000000000..e38d3e9a6f --- /dev/null +++ b/bin/patch-create-missing-schedules @@ -0,0 +1,3 @@ +#!/bin/sh + +php /usr/src/code/app/cli.php patch-create-missing-schedules $@ \ No newline at end of file diff --git a/src/Appwrite/Platform/Services/Tasks.php b/src/Appwrite/Platform/Services/Tasks.php index e6919f2cc7..2e15cd015c 100644 --- a/src/Appwrite/Platform/Services/Tasks.php +++ b/src/Appwrite/Platform/Services/Tasks.php @@ -8,7 +8,7 @@ use Appwrite\Platform\Tasks\Install; use Appwrite\Platform\Tasks\Maintenance; use Appwrite\Platform\Tasks\Migrate; use Appwrite\Platform\Tasks\Schedule; -use Appwrite\Platform\Tasks\CreateMissingSchedules; +use Appwrite\Platform\Tasks\PatchCreateMissingSchedules; use Appwrite\Platform\Tasks\SDKs; use Appwrite\Platform\Tasks\Specs; use Appwrite\Platform\Tasks\SSL; @@ -30,7 +30,7 @@ class Tasks extends Service ->addAction(Doctor::getName(), new Doctor()) ->addAction(Install::getName(), new Install()) ->addAction(Maintenance::getName(), new Maintenance()) - ->addAction(CreateMissingSchedules::getName(), new CreateMissingSchedules()) + ->addAction(PatchCreateMissingSchedules::getName(), new PatchCreateMissingSchedules()) ->addAction(Schedule::getName(), new Schedule()) ->addAction(Migrate::getName(), new Migrate()) ->addAction(SDKs::getName(), new SDKs()) diff --git a/src/Appwrite/Platform/Tasks/CreateMissingSchedules.php b/src/Appwrite/Platform/Tasks/PatchCreateMissingSchedules.php similarity index 93% rename from src/Appwrite/Platform/Tasks/CreateMissingSchedules.php rename to src/Appwrite/Platform/Tasks/PatchCreateMissingSchedules.php index 75e1d5b637..32b9886347 100644 --- a/src/Appwrite/Platform/Tasks/CreateMissingSchedules.php +++ b/src/Appwrite/Platform/Tasks/PatchCreateMissingSchedules.php @@ -11,11 +11,11 @@ use Utopia\Database\Database; use Utopia\Database\ID; use Utopia\Database\Validator\Authorization; -class CreateMissingSchedules extends Action +class PatchCreateMissingSchedules extends Action { public static function getName(): string { - return 'create-missing-schedules'; + return 'patch-create-missing-schedules'; } public function __construct() @@ -35,8 +35,8 @@ class CreateMissingSchedules extends Action Authorization::disable(); Authorization::setDefaultStatus(false); - Console::title('CreateMissingSchedules V1'); - Console::success(APP_NAME . ' CreateMissingSchedules v1 has started'); + Console::title('PatchCreateMissingSchedules V1'); + Console::success(APP_NAME . ' PatchCreateMissingSchedules v1 has started'); $limit = 100; $projectCursor = null; From 3af214172cc8417289052494949baa671ee7cf47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Wed, 14 Dec 2022 11:10:16 +0100 Subject: [PATCH 5/5] Rollback unwanted changes --- .env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env b/.env index fee6c6653d..51e6cc3705 100644 --- a/.env +++ b/.env @@ -64,7 +64,7 @@ _APP_MAINTENANCE_RETENTION_CACHE=2592000 _APP_MAINTENANCE_RETENTION_EXECUTION=1209600 _APP_MAINTENANCE_RETENTION_ABUSE=86400 _APP_MAINTENANCE_RETENTION_AUDIT=1209600 -_APP_MAINTENANCE_RETENTION_SCHEDULES=600 +_APP_MAINTENANCE_RETENTION_SCHEDULES=86400 _APP_MAINTENANCE_RETENTION_USAGE_HOURLY=8640000 _APP_USAGE_STATS=enabled _APP_USAGE_AGGREGATION_INTERVAL=30