From 546d8ba5834d8f12e01c60dd9e43b1380b29886d Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 31 Oct 2022 07:31:31 +0000 Subject: [PATCH 01/25] refactor to use db pools --- app/tasks/usage.php | 9 +++-- src/Appwrite/Usage/Calculators/TimeSeries.php | 34 +++++++++++++++---- 2 files changed, 31 insertions(+), 12 deletions(-) diff --git a/app/tasks/usage.php b/app/tasks/usage.php index 68543d0748..c9596e6c65 100644 --- a/app/tasks/usage.php +++ b/app/tasks/usage.php @@ -3,13 +3,11 @@ global $cli, $register; use Appwrite\Usage\Calculators\TimeSeries; -use InfluxDB\Database as InfluxDatabase; use Utopia\App; use Utopia\CLI\Console; -use Utopia\Database\Database as UtopiaDatabase; +use Utopia\Database\Document; use Utopia\Database\Validator\Authorization; use Utopia\Logger\Log; -use Utopia\Validator\WhiteList; Authorization::disable(); Authorization::setDefaultStatus(false); @@ -51,15 +49,16 @@ $logError = function (Throwable $error, string $action = 'syncUsageStats') use ( $cli ->task('usage') ->desc('Schedules syncing data from influxdb to Appwrite console db') - ->action(function () use ($logError) { + ->action(function () use ($register, $logError) { Console::title('Usage Aggregation V1'); Console::success(APP_NAME . ' usage aggregation process v1 has started'); $database = getConsoleDB(); $influxDB = getInfluxDB(); + $getProjectDB = fn (Document $project) => getProjectDB($project); $interval = (int) App::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '30'); // 30 seconds (by default) - $usage = new TimeSeries($database, $influxDB, $logError); + $usage = new TimeSeries($database, $influxDB, $getProjectDB, $register, $logError); Console::loop(function () use ($interval, $usage) { $now = date('d-m-Y H:i:s', time()); diff --git a/src/Appwrite/Usage/Calculators/TimeSeries.php b/src/Appwrite/Usage/Calculators/TimeSeries.php index 20308684de..6098f55509 100644 --- a/src/Appwrite/Usage/Calculators/TimeSeries.php +++ b/src/Appwrite/Usage/Calculators/TimeSeries.php @@ -7,6 +7,7 @@ use Utopia\Database\Database; use Utopia\Database\Document; use InfluxDB\Database as InfluxDatabase; use DateTime; +use Utopia\Registry\Registry; class TimeSeries extends Calculator { @@ -31,6 +32,20 @@ class TimeSeries extends Calculator */ protected $errorHandler; + /** + * Callback to get project DB + * + * @var callable + */ + protected mixed $getProjectDB; + + /** + * Registry + * + * @var Registry + */ + protected Registry $register; + /** * Latest times for metric that was synced to the database * @@ -381,10 +396,12 @@ class TimeSeries extends Calculator ] ]; - public function __construct(Database $database, InfluxDatabase $influxDB, callable $errorHandler = null) + public function __construct(Database $database, InfluxDatabase $influxDB, callable $getProjectDB, Registry $register, callable $errorHandler = null) { $this->database = $database; $this->influxDB = $influxDB; + $this->getProjectDB = $getProjectDB; + $this->register = $register; $this->errorHandler = $errorHandler; } @@ -404,12 +421,13 @@ class TimeSeries extends Calculator private function createOrUpdateMetric(string $projectId, string $time, string $period, string $metric, int $value, int $type): void { $id = \md5("{$time}_{$period}_{$metric}"); - $this->database->setNamespace('_' . $projectId); + $project = $this->database->getDocument('projects', $projectId); + $database = call_user_func($this->getProjectDB, $project); try { - $document = $this->database->getDocument('stats', $id); + $document = $database->getDocument('stats', $id); if ($document->isEmpty()) { - $this->database->createDocument('stats', new Document([ + $database->createDocument('stats', new Document([ '$id' => $id, 'period' => $period, 'time' => $time, @@ -418,7 +436,7 @@ class TimeSeries extends Calculator 'type' => $type, ])); } else { - $this->database->updateDocument( + $database->updateDocument( 'stats', $document->getId(), $document->setAttribute('value', $value) @@ -431,6 +449,8 @@ class TimeSeries extends Calculator throw $e; } } + + $this->register->get('pools')->reclaim(); } /** @@ -466,7 +486,7 @@ class TimeSeries extends Calculator $query .= "WHERE \"time\" > '{$start}' "; $query .= "AND \"time\" < '{$end}' "; $query .= "AND \"metric_type\"='counter' {$filters} "; - $query .= "GROUP BY time({$period['key']}), \"projectId\", \"projectInternalId\" {$groupBy} "; + $query .= "GROUP BY time({$period['key']}), \"projectId\" {$groupBy} "; $query .= "FILL(null)"; try { @@ -490,7 +510,7 @@ class TimeSeries extends Calculator $value = (!empty($point['value'])) ? $point['value'] : 0; $this->createOrUpdateMetric( - $point['projectInternalId'], + $point['projectId'], $point['time'], $period['key'], $metricUpdated, From 1db29e7674a685cbb5f1162b587beba923264a73 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 31 Oct 2022 08:42:33 +0000 Subject: [PATCH 02/25] appwrite logs --- .github/workflows/tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1d1d10c8a1..f5da75777d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -31,6 +31,8 @@ jobs: docker compose build appwrite docker compose up -d sleep 30 + - name: logs + run: docker compose logs appwrite - name: Doctor run: docker compose exec -T appwrite doctor From 86d377722bce9c957cc36db4c7bd34ed4c6823ba Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 31 Oct 2022 10:35:33 +0000 Subject: [PATCH 03/25] composer update --- composer.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/composer.lock b/composer.lock index 86d47f4a19..47b30929f8 100644 --- a/composer.lock +++ b/composer.lock @@ -5152,5 +5152,5 @@ "platform-overrides": { "php": "8.0" }, - "plugin-api-version": "2.2.0" + "plugin-api-version": "2.3.0" } From 3f3aea37d8ea459ccc76fe824545a9311bb2e898 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 1 Nov 2022 06:08:10 +0000 Subject: [PATCH 04/25] update utopia libraries --- composer.json | 6 +++--- composer.lock | 57 ++++++++++++++++++++++----------------------------- 2 files changed, 27 insertions(+), 36 deletions(-) diff --git a/composer.json b/composer.json index 1748f7cac6..29a351aa1e 100644 --- a/composer.json +++ b/composer.json @@ -45,13 +45,13 @@ "appwrite/php-runtimes": "0.11.*", "utopia-php/framework": "0.21.*", "utopia-php/logger": "0.3.*", - "utopia-php/abuse": "0.15.*", + "utopia-php/abuse": "0.16.*", "utopia-php/analytics": "0.2.*", - "utopia-php/audit": "0.16.*", + "utopia-php/audit": "0.17.*", "utopia-php/cache": "0.8.*", "utopia-php/cli": "0.13.*", "utopia-php/config": "0.2.*", - "utopia-php/database": "dev-feat-update-create as 0.27.0", + "utopia-php/database": "0.28.*", "utopia-php/locale": "0.4.*", "utopia-php/registry": "0.5.*", "utopia-php/preloader": "0.2.*", diff --git a/composer.lock b/composer.lock index 47b30929f8..5a81dafebd 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "c33b697a2f1422eb6fccfd860a7e0169", + "content-hash": "e44426d5884ef97987ebbb711a9ee189", "packages": [ { "name": "adhocore/jwt", @@ -1462,23 +1462,23 @@ }, { "name": "utopia-php/abuse", - "version": "0.15.0", + "version": "0.16.0", "source": { "type": "git", "url": "https://github.com/utopia-php/abuse.git", - "reference": "fa7684d3162d370cf17df349066dd6535dd7264a" + "reference": "6370d9150425460416583feba0990504ac789e98" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/abuse/zipball/fa7684d3162d370cf17df349066dd6535dd7264a", - "reference": "fa7684d3162d370cf17df349066dd6535dd7264a", + "url": "https://api.github.com/repos/utopia-php/abuse/zipball/6370d9150425460416583feba0990504ac789e98", + "reference": "6370d9150425460416583feba0990504ac789e98", "shasum": "" }, "require": { "ext-curl": "*", "ext-pdo": "*", "php": ">=8.0", - "utopia-php/database": "0.27.*" + "utopia-php/database": "0.28.*" }, "require-dev": { "phpunit/phpunit": "^9.4", @@ -1510,9 +1510,9 @@ ], "support": { "issues": "https://github.com/utopia-php/abuse/issues", - "source": "https://github.com/utopia-php/abuse/tree/0.15.0" + "source": "https://github.com/utopia-php/abuse/tree/0.16.0" }, - "time": "2022-10-30T08:11:12+00:00" + "time": "2022-10-31T14:46:41+00:00" }, { "name": "utopia-php/analytics", @@ -1571,22 +1571,22 @@ }, { "name": "utopia-php/audit", - "version": "0.16.0", + "version": "0.17.0", "source": { "type": "git", "url": "https://github.com/utopia-php/audit.git", - "reference": "fd8a0ea973f50443e6b7b8d0728d4c1e136da0ef" + "reference": "455471bd4de8d74026809e843f8c9740eb32922c" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/audit/zipball/fd8a0ea973f50443e6b7b8d0728d4c1e136da0ef", - "reference": "fd8a0ea973f50443e6b7b8d0728d4c1e136da0ef", + "url": "https://api.github.com/repos/utopia-php/audit/zipball/455471bd4de8d74026809e843f8c9740eb32922c", + "reference": "455471bd4de8d74026809e843f8c9740eb32922c", "shasum": "" }, "require": { "ext-pdo": "*", "php": ">=8.0", - "utopia-php/database": "0.27.*" + "utopia-php/database": "0.28.*" }, "require-dev": { "phpunit/phpunit": "^9.3", @@ -1612,9 +1612,9 @@ ], "support": { "issues": "https://github.com/utopia-php/audit/issues", - "source": "https://github.com/utopia-php/audit/tree/0.16.0" + "source": "https://github.com/utopia-php/audit/tree/0.17.0" }, - "time": "2022-10-30T08:11:26+00:00" + "time": "2022-10-31T14:44:52+00:00" }, { "name": "utopia-php/cache", @@ -1771,16 +1771,16 @@ }, { "name": "utopia-php/database", - "version": "dev-feat-update-create", + "version": "0.28.0", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "8ba52f4ce40674ed9f6e8f72e300d5d9a4900590" + "reference": "ef6506af1c09c22f5dc1e7859159d323f7fafa94" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/8ba52f4ce40674ed9f6e8f72e300d5d9a4900590", - "reference": "8ba52f4ce40674ed9f6e8f72e300d5d9a4900590", + "url": "https://api.github.com/repos/utopia-php/database/zipball/ef6506af1c09c22f5dc1e7859159d323f7fafa94", + "reference": "ef6506af1c09c22f5dc1e7859159d323f7fafa94", "shasum": "" }, "require": { @@ -1819,9 +1819,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/feat-update-create" + "source": "https://github.com/utopia-php/database/tree/0.28.0" }, - "time": "2022-10-28T12:01:54+00:00" + "time": "2022-10-31T09:58:46+00:00" }, { "name": "utopia-php/domains", @@ -5117,18 +5117,9 @@ "time": "2022-09-28T08:42:51+00:00" } ], - "aliases": [ - { - "package": "utopia-php/database", - "version": "dev-feat-update-create", - "alias": "0.27.0", - "alias_normalized": "0.27.0.0" - } - ], + "aliases": [], "minimum-stability": "stable", - "stability-flags": { - "utopia-php/database": 20 - }, + "stability-flags": [], "prefer-stable": false, "prefer-lowest": false, "platform": { @@ -5152,5 +5143,5 @@ "platform-overrides": { "php": "8.0" }, - "plugin-api-version": "2.3.0" + "plugin-api-version": "2.2.0" } From fee0ec960927da52183d641be719d41fc5f9b7bf Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Wed, 2 Nov 2022 01:35:07 +0000 Subject: [PATCH 05/25] remove vendor framework mount --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 081d330f1a..1ab2776efb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -84,7 +84,7 @@ services: - ./public:/usr/src/code/public - ./src:/usr/src/code/src - ./dev:/usr/local/dev - - ./vendor/utopia-php/framework:/usr/src/code/vendor/utopia-php/framework + depends_on: - mariadb - redis From 777ed70669cf53ead2fce2a02e1ac38ab10c1441 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Wed, 2 Nov 2022 05:35:54 +0000 Subject: [PATCH 06/25] composer update --- composer.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/composer.lock b/composer.lock index 5a81dafebd..3494350421 100644 --- a/composer.lock +++ b/composer.lock @@ -5143,5 +5143,5 @@ "platform-overrides": { "php": "8.0" }, - "plugin-api-version": "2.2.0" + "plugin-api-version": "2.3.0" } From 85920e5b147cbdf5008894c5ad55183188b58a81 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Wed, 2 Nov 2022 05:38:46 +0000 Subject: [PATCH 07/25] fix missing mariadb --- app/controllers/api/databases.php | 1 + 1 file changed, 1 insertion(+) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index 6dbb607ce4..ab62ba15a9 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -48,6 +48,7 @@ use Appwrite\Utopia\Database\Validator\Queries\Databases; use Appwrite\Utopia\Database\Validator\Queries\Documents; use Utopia\Config\Config; use MaxMind\Db\Reader; +use Utopia\Database\Adapter\MariaDB; /** * Create attribute of varying type From fb120be78ee8397a5a0b36557a434b4f5fadd71d Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Wed, 2 Nov 2022 05:48:17 +0000 Subject: [PATCH 08/25] get projectdb on CLI --- app/cli.php | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/app/cli.php b/app/cli.php index 3a62c80816..248eec3353 100644 --- a/app/cli.php +++ b/app/cli.php @@ -12,6 +12,7 @@ use Utopia\Config\Config; use Utopia\Database\Database; use Utopia\Database\Validator\Authorization; use InfluxDB\Database as InfluxDatabase; +use Utopia\Database\Document; function getInfluxDB(): InfluxDatabase { @@ -59,6 +60,28 @@ function getConsoleDB(): Database return $database; } +function getProjectDB(Document $project): Database +{ + global $register; + + $pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */ + + if ($project->isEmpty() || $project->getId() === 'console') { + return getConsoleDB(); + } + + $dbAdapter = $pools + ->get($project->getAttribute('database')) + ->pop() + ->getResource() + ; + + $database = new Database($dbAdapter, getCache()); + $database->setNamespace('_' . $project->getInternalId()); + + return $database; +} + function getCache(): Cache { global $register; From 74eaef4e9d7074953bd76bb9ac0be07b3a56b82f Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 17 Nov 2022 05:51:12 +0545 Subject: [PATCH 09/25] remove unused import --- app/controllers/api/databases.php | 1 - 1 file changed, 1 deletion(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index 2a56fe0183..5022681d87 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -47,7 +47,6 @@ use Appwrite\Utopia\Database\Validator\Queries\Databases; use Appwrite\Utopia\Database\Validator\Queries\Documents; use Utopia\Config\Config; use MaxMind\Db\Reader; -use Utopia\Database\Adapter\MariaDB; /** * Create attribute of varying type From 9f92214c9d1a12d822a5f3ed7b09c6999062c03d Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 17 Nov 2022 00:36:51 +0000 Subject: [PATCH 10/25] update usage --- src/Appwrite/Platform/Tasks/Usage.php | 56 +++++++-------------------- 1 file changed, 15 insertions(+), 41 deletions(-) diff --git a/src/Appwrite/Platform/Tasks/Usage.php b/src/Appwrite/Platform/Tasks/Usage.php index 73686109d8..6cf20e737d 100644 --- a/src/Appwrite/Platform/Tasks/Usage.php +++ b/src/Appwrite/Platform/Tasks/Usage.php @@ -12,6 +12,7 @@ use Utopia\Database\Database as UtopiaDatabase; use Utopia\Validator\WhiteList; use Throwable; use Utopia\Platform\Action; +use Utopia\Registry\Registry; class Usage extends Action { @@ -24,17 +25,28 @@ class Usage extends Action { $this ->desc('Schedules syncing data from influxdb to Appwrite console db') - ->param('type', 'timeseries', new WhiteList(['timeseries', 'database'])) ->inject('dbForConsole') ->inject('influxdb') + ->inject('register') + ->inject('getProjectDB') ->inject('logError') - ->callback(fn ($type, $dbForConsole, $influxDB, $logError) => $this->action($type, $dbForConsole, $influxDB, $logError)); + ->callback(fn ($dbForConsole, $influxDB, $register, $getProjectDB, $logError) => $this->action($dbForConsole, $influxDB, $register, $getProjectDB, $logError)); } protected function aggregateTimeseries(UtopiaDatabase $database, InfluxDatabase $influxDB, callable $logError): void { + + } + + public function action(UtopiaDatabase $dbForConsole, InfluxDatabase $influxDB, Registry $register, callable $getProjectDB, callable $logError) + { + Console::title('Usage Aggregation V1'); + Console::success(APP_NAME . ' usage aggregation process v1 has started'); + + $errorLogger = fn(Throwable $error, string $action = 'syncUsageStats') => $logError($error, "usage", $action); + $interval = (int) App::getEnv('_APP_USAGE_TIMESERIES_INTERVAL', '30'); // 30 seconds (by default) - $usage = new TimeSeries($database, $influxDB, $logError); + $usage = new TimeSeries($dbForConsole, $influxDB, $getProjectDB, $register, $errorLogger); Console::loop(function () use ($interval, $usage) { $now = date('d-m-Y H:i:s', time()); @@ -48,42 +60,4 @@ class Usage extends Action Console::info("[{$now}] Aggregation took {$loopTook} seconds"); }, $interval); } - - protected function aggregateDatabase(UtopiaDatabase $database, callable $logError): void - { - $interval = (int) App::getEnv('_APP_USAGE_DATABASE_INTERVAL', '900'); // 15 minutes (by default) - $usage = new Database($database, $logError); - $aggregrator = new Aggregator($database, $logError); - - Console::loop(function () use ($interval, $usage, $aggregrator) { - $now = date('d-m-Y H:i:s', time()); - Console::info("[{$now}] Aggregating database usage every {$interval} seconds."); - $loopStart = microtime(true); - $usage->collect(); - $aggregrator->collect(); - $loopTook = microtime(true) - $loopStart; - $now = date('d-m-Y H:i:s', time()); - - Console::info("[{$now}] Aggregation took {$loopTook} seconds"); - }, $interval); - } - - public function action(string $type, UtopiaDatabase $dbForConsole, InfluxDatabase $influxDB, callable $logError) - { - Console::title('Usage Aggregation V1'); - Console::success(APP_NAME . ' usage aggregation process v1 has started'); - - $errorLogger = fn(Throwable $error, string $action = 'syncUsageStats') => $logError($error, "usage", $action); - - switch ($type) { - case 'timeseries': - $this->aggregateTimeseries($dbForConsole, $influxDB, $errorLogger); - break; - case 'database': - $this->aggregateDatabase($dbForConsole, $errorLogger); - break; - default: - Console::error("Unsupported usage aggregation type"); - } - } } From baddbf7f03ced61c2ccfaf02267fff0113bf9beb Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 17 Nov 2022 00:38:44 +0000 Subject: [PATCH 11/25] remove unused code --- src/Appwrite/Platform/Tasks/Usage.php | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/Appwrite/Platform/Tasks/Usage.php b/src/Appwrite/Platform/Tasks/Usage.php index 6cf20e737d..d083839145 100644 --- a/src/Appwrite/Platform/Tasks/Usage.php +++ b/src/Appwrite/Platform/Tasks/Usage.php @@ -2,14 +2,11 @@ namespace Appwrite\Platform\Tasks; -use Appwrite\Usage\Calculators\Aggregator; -use Appwrite\Usage\Calculators\Database; use Appwrite\Usage\Calculators\TimeSeries; use InfluxDB\Database as InfluxDatabase; use Utopia\App; use Utopia\CLI\Console; use Utopia\Database\Database as UtopiaDatabase; -use Utopia\Validator\WhiteList; use Throwable; use Utopia\Platform\Action; use Utopia\Registry\Registry; From 3439978253a348c258316bb87c0dc9fd6873be84 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 17 Nov 2022 00:39:33 +0000 Subject: [PATCH 12/25] formatting --- src/Appwrite/Platform/Tasks/Usage.php | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Appwrite/Platform/Tasks/Usage.php b/src/Appwrite/Platform/Tasks/Usage.php index d083839145..1315d71c39 100644 --- a/src/Appwrite/Platform/Tasks/Usage.php +++ b/src/Appwrite/Platform/Tasks/Usage.php @@ -32,7 +32,6 @@ class Usage extends Action protected function aggregateTimeseries(UtopiaDatabase $database, InfluxDatabase $influxDB, callable $logError): void { - } public function action(UtopiaDatabase $dbForConsole, InfluxDatabase $influxDB, Registry $register, callable $getProjectDB, callable $logError) From 9698dc2115b8904cf08553c726e12d04d485c5c5 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Fri, 18 Nov 2022 22:18:27 +0530 Subject: [PATCH 13/25] fix: volume sync --- app/cli.php | 89 ++++++++++++++++++---------------------------- app/console | 2 +- docker-compose.yml | 13 +++++++ 3 files changed, 49 insertions(+), 55 deletions(-) diff --git a/app/cli.php b/app/cli.php index 9857261b47..bfe7bfcefb 100644 --- a/app/cli.php +++ b/app/cli.php @@ -43,17 +43,44 @@ CLI::setResource('pools', function (Registry $register) { }, ['register']); CLI::setResource('dbForConsole', function ($pools, $cache) { - $dbAdapter = $pools - ->get('console') - ->pop() - ->getResource() - ; + $sleep = 3; + $maxAttempts = 5; + $attempts = 0; + $ready = false; - $database = new Database($dbAdapter, $cache); + do { + $attempts++; + try { + // Prepare database connection + $dbAdapter = $pools + ->get('console') + ->pop() + ->getResource(); - $database->setNamespace('console'); + $dbForConsole = new Database($dbAdapter, $cache); + $dbForConsole->setNamespace('console'); - return $database; + // Ensure tables exist + $collections = Config::getParam('collections', []); + $last = \array_key_last($collections); + + if (!($dbForConsole->exists($dbForConsole->getDefaultDatabase(), $last))) { /** TODO cache ready variable using registry */ + throw new Exception('Tables not ready yet.'); + } + + $ready = true; + } catch (\Exception $err) { + Console::warning($err->getMessage()); + $pools->get('console')->reclaim(); + sleep($sleep); + } + } while ($attempts < $maxAttempts); + + if (!$ready) { + throw new Exception("Console is not ready yet. Please try again later."); + } + + return $dbForConsole; }, ['pools', 'cache']); CLI::setResource('getProjectDB', function (Group $pools, Database $dbForConsole, $cache) { @@ -165,50 +192,4 @@ $cli Console::error($error->getMessage()); }); -$cli - ->init() - ->inject('pools') - ->inject('cache') - ->action(function (Group $pools, Cache $cache) { - $maxAttempts = 5; - $sleep = 3; - - $attempts = 0; - $ready = false; - - do { - $attempts++; - - try { - $pools->get('console')->reclaim(); - - // Prepare database connection - $dbAdapter = $pools - ->get('console') - ->pop() - ->getResource(); - - $dbForConsole = new Database($dbAdapter, $cache); - $dbForConsole->setNamespace('console'); - - // Ensure tables exist - $collections = Config::getParam('collections', []); - $last = \array_key_last($collections); - - if (!($dbForConsole->exists($dbForConsole->getDefaultDatabase(), $last))) { - throw new Exception('Tables not ready yet.'); - } - - $ready = true; - } catch (\Exception $err) { - Console::warning($err->getMessage()); - sleep($sleep); - } - } while ($attempts < $maxAttempts); - - if (!$ready) { - throw new Exception("Console is not ready yet. Please try again later."); - } - }); - $cli->run(); diff --git a/app/console b/app/console index 6aa50b8b96..4e2cecefb5 160000 --- a/app/console +++ b/app/console @@ -1 +1 @@ -Subproject commit 6aa50b8b96b7941a710fe097f3f4d3a74f1eb823 +Subproject commit 4e2cecefb571104f0dbbe5a578729f0e17a10242 diff --git a/docker-compose.yml b/docker-compose.yml index 4ca8876f9b..cb18dff4ea 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -769,6 +769,19 @@ services: - 9506:8080 networks: - appwrite + + # appwrite-volume-sync: + # entrypoint: volume-sync + # <<: *x-logging + # container_name: appwrite-volume-sync + # image: appwrite-dev + # command: + # - --source=/data/src/ --destination=/data/dest/ --interval=10 + # networks: + # - appwrite + # # volumes: # Mount the rsync source and destination directories + # # - /nfs/config:/data/src + # # - /storage/config:/data/dest # redis-commander: # image: rediscommander/redis-commander:latest From 202e4c01d8f4054b9b359892e2d4110296d6f7e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Mon, 21 Nov 2022 14:23:43 +0000 Subject: [PATCH 14/25] Add new executor env vars --- .env | 1 + app/config/variables.php | 11 ++++++++++- app/views/install/compose.phtml | 1 + docker-compose.yml | 1 + 4 files changed, 13 insertions(+), 1 deletion(-) diff --git a/.env b/.env index 15a25aa933..4656731909 100644 --- a/.env +++ b/.env @@ -53,6 +53,7 @@ _APP_FUNCTIONS_BUILD_TIMEOUT=900 _APP_FUNCTIONS_CPUS=1 _APP_FUNCTIONS_MEMORY=512 _APP_FUNCTIONS_INACTIVE_THRESHOLD=600 +_APP_FUNCTIONS_MAINTENANCE_INTERVAL=600 _APP_FUNCTIONS_RUNTIMES_NETWORK=runtimes _APP_EXECUTOR_SECRET=your-secret-key _APP_EXECUTOR_HOST=http://exc1/v1 diff --git a/app/config/variables.php b/app/config/variables.php index 1eb2688c62..474ec70d66 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -877,7 +877,16 @@ return [ 'required' => false, 'question' => '', 'filter' => '' - ] + ], + [ + 'name' => '_APP_FUNCTIONS_MAINTENANCE_INTERVAL', + 'description' => 'Interval how often executor checks for inactive runimes. The default value is 60 seconds.', + 'introduction' => '1.2.0', + 'default' => '60', + 'required' => false, + 'question' => '', + 'filter' => '' + ], ], ], [ diff --git a/app/views/install/compose.phtml b/app/views/install/compose.phtml index dc7549aa3f..c1939c46c2 100644 --- a/app/views/install/compose.phtml +++ b/app/views/install/compose.phtml @@ -558,6 +558,7 @@ services: environment: - OPR_EXECUTOR_CONNECTION_STORAGE=$_APP_CONNECTIONS_STORAGE - OPR_EXECUTOR_INACTIVE_TRESHOLD=$_APP_FUNCTIONS_INACTIVE_THRESHOLD + - OPR_EXECUTOR_MAINTENANCE_INTERVAL=$_APP_FUNCTIONS_MAINTENANCE_INTERVAL - OPR_EXECUTOR_NETWORK=$_APP_FUNCTIONS_RUNTIMES_NETWORK - OPR_EXECUTOR_DOCKER_HUB_USERNAME=$_APP_DOCKER_HUB_USERNAME - OPR_EXECUTOR_DOCKER_HUB_PASSWORD=$_APP_DOCKER_HUB_PASSWORD diff --git a/docker-compose.yml b/docker-compose.yml index cb18dff4ea..82b85ba733 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -650,6 +650,7 @@ services: environment: - OPR_EXECUTOR_CONNECTION_STORAGE=$_APP_CONNECTIONS_STORAGE - OPR_EXECUTOR_INACTIVE_TRESHOLD=$_APP_FUNCTIONS_INACTIVE_THRESHOLD + - OPR_EXECUTOR_MAINTENANCE_INTERVAL=$_APP_FUNCTIONS_MAINTENANCE_INTERVAL - OPR_EXECUTOR_NETWORK=$_APP_FUNCTIONS_RUNTIMES_NETWORK - OPR_EXECUTOR_DOCKER_HUB_USERNAME=$_APP_DOCKER_HUB_USERNAME - OPR_EXECUTOR_DOCKER_HUB_PASSWORD=$_APP_DOCKER_HUB_PASSWORD From e20b2f33403a6c246c02506f42dd59366eefb432 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Mon, 21 Nov 2022 14:55:57 +0000 Subject: [PATCH 15/25] Upgrade OPR images --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 82b85ba733..59661a15c5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -638,7 +638,7 @@ services: hostname: exc1 <<: *x-logging stop_signal: SIGINT - image: openruntimes/executor:0.1.4 + image: openruntimes/executor:0.1.6 networks: - appwrite - runtimes From f055adc05f9bbe95becaec5d69cf2181db18667c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Tue, 22 Nov 2022 07:17:56 +0000 Subject: [PATCH 16/25] Fix pool sharing logic --- .env | 3 ++- app/init.php | 14 ++++++++++---- docker-compose.yml | 2 ++ 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/.env b/.env index 4656731909..b8bc2d93cf 100644 --- a/.env +++ b/.env @@ -24,7 +24,8 @@ _APP_DB_SCHEMA=appwrite _APP_DB_USER=user _APP_DB_PASS=password _APP_DB_ROOT_PASS=rootsecretpassword -_APP_CONNECTIONS_MAX=251 +_APP_CONNECTIONS_MAX=151 +_APP_POOL_CLIENTS=14 _APP_CONNECTIONS_DB_PROJECT=db_fra1_02=mariadb://user:password@mariadb:3306/appwrite _APP_CONNECTIONS_DB_CONSOLE=db_fra1_01=mariadb://user:password@mariadb:3306/appwrite _APP_CONNECTIONS_CACHE=redis_fra1_01=redis://redis:6379 diff --git a/app/init.php b/app/init.php index 5016a4246c..04e5387381 100644 --- a/app/init.php +++ b/app/init.php @@ -552,10 +552,16 @@ $register->set('pools', function () { ], ]; - $instances = 3; // REST, Realtime, CLI - $workerCount = swoole_cpu_num() * intval(App::getEnv('_APP_WORKER_PER_CORE', 6)); - $maxConnections = App::getenv('_APP_CONNECTIONS_MAX', 251); - $instanceConnections = $maxConnections / $instances; + $maxConnections = App::getenv('_APP_CONNECTIONS_MAX', 151); + $instanceConnections = $maxConnections / App::getenv('_APP_POOL_CLIENTS', 14); + + $multiprocessing = App::getenv('_APP_SERVER_MULTIPROCESS', 'disabled') === 'enabled'; + + if($multiprocessing) { + $workerCount = swoole_cpu_num() * intval(App::getEnv('_APP_WORKER_PER_CORE', 6)); + } else { + $workerCount = 1; + } if ($workerCount > $instanceConnections) { throw new \Exception('Pool size is too small. Increase the number of allowed database connections or decrease the number of workers.', 500); diff --git a/docker-compose.yml b/docker-compose.yml index 59661a15c5..59907aaf04 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -105,6 +105,7 @@ services: environment: - _APP_ENV - _APP_WORKER_PER_CORE + - _APP_SERVER_MULTIPROCESS=enabled - _APP_LOCALE - _APP_CONSOLE_WHITELIST_ROOT - _APP_CONSOLE_WHITELIST_EMAILS @@ -204,6 +205,7 @@ services: environment: - _APP_ENV - _APP_WORKER_PER_CORE + - _APP_SERVER_MULTIPROCESS=enabled - _APP_OPTIONS_ABUSE - _APP_OPENSSL_KEY_V1 - _APP_DB_HOST From ced481f78c22bb7ee74038cddb13a56fd7badc4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Tue, 22 Nov 2022 07:57:44 +0000 Subject: [PATCH 17/25] Linter, library upgrade --- app/init.php | 2 +- composer.lock | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/app/init.php b/app/init.php index 04e5387381..550dc6d744 100644 --- a/app/init.php +++ b/app/init.php @@ -557,7 +557,7 @@ $register->set('pools', function () { $multiprocessing = App::getenv('_APP_SERVER_MULTIPROCESS', 'disabled') === 'enabled'; - if($multiprocessing) { + if ($multiprocessing) { $workerCount = swoole_cpu_num() * intval(App::getEnv('_APP_WORKER_PER_CORE', 6)); } else { $workerCount = 1; diff --git a/composer.lock b/composer.lock index f47d853d6b..c41db9e924 100644 --- a/composer.lock +++ b/composer.lock @@ -2356,16 +2356,16 @@ }, { "name": "utopia-php/pools", - "version": "0.4.1", + "version": "0.4.2", "source": { "type": "git", "url": "https://github.com/utopia-php/pools.git", - "reference": "c8f96a33e7fbf58c1145eb6cf0f2c00cbe319979" + "reference": "d2870ab74b31b7f4027799f082e85122154f8bed" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/pools/zipball/c8f96a33e7fbf58c1145eb6cf0f2c00cbe319979", - "reference": "c8f96a33e7fbf58c1145eb6cf0f2c00cbe319979", + "url": "https://api.github.com/repos/utopia-php/pools/zipball/d2870ab74b31b7f4027799f082e85122154f8bed", + "reference": "d2870ab74b31b7f4027799f082e85122154f8bed", "shasum": "" }, "require": { @@ -2401,9 +2401,9 @@ ], "support": { "issues": "https://github.com/utopia-php/pools/issues", - "source": "https://github.com/utopia-php/pools/tree/0.4.1" + "source": "https://github.com/utopia-php/pools/tree/0.4.2" }, - "time": "2022-11-15T08:55:16+00:00" + "time": "2022-11-22T07:55:45+00:00" }, { "name": "utopia-php/preloader", @@ -5403,5 +5403,5 @@ "platform-overrides": { "php": "8.0" }, - "plugin-api-version": "2.1.0" + "plugin-api-version": "2.3.0" } From 8ea6647b22feaf8d55dbf2f07d8ec477eabc2bc4 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Tue, 22 Nov 2022 13:56:50 +0000 Subject: [PATCH 18/25] feat: update branch --- .env | 4 +- .github/workflows/tests.yml | 3 +- app/console | 2 +- app/controllers/api/projects.php | 112 ------------------------------- composer.json | 10 +-- composer.lock | 14 ++-- 6 files changed, 16 insertions(+), 129 deletions(-) diff --git a/.env b/.env index 4e86263840..9cac2e6438 100644 --- a/.env +++ b/.env @@ -62,10 +62,10 @@ _APP_MAINTENANCE_RETENTION_CACHE=2592000 _APP_MAINTENANCE_RETENTION_EXECUTION=1209600 _APP_MAINTENANCE_RETENTION_ABUSE=86400 _APP_MAINTENANCE_RETENTION_AUDIT=1209600 -_APP_MAINTENANCE_RETENTION_SCHEDULES= 86400 -_APP_USAGE_AGGREGATION_INTERVAL=5 +_APP_MAINTENANCE_RETENTION_SCHEDULES=86400 _APP_MAINTENANCE_RETENTION_USAGE_HOURLY=8640000 _APP_USAGE_STATS=enabled +_APP_USAGE_AGGREGATION_INTERVAL=5 _APP_LOGGING_PROVIDER= _APP_LOGGING_CONFIG= _APP_REGION=default diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index cd9faa39c4..cff75b73b4 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -33,8 +33,7 @@ jobs: docker compose build appwrite docker compose up -d sleep 30 - - name: logs - run: docker compose logs appwrite + - name: Doctor run: docker compose exec -T appwrite doctor diff --git a/app/console b/app/console index 4e2cecefb5..25d281795a 160000 --- a/app/console +++ b/app/console @@ -1 +1 @@ -Subproject commit 4e2cecefb571104f0dbbe5a578729f0e17a10242 +Subproject commit 25d281795a1b73c9b7ed37b14c451068c36d2ff1 diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index f6d0a58c19..ac2bb675b5 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -28,7 +28,6 @@ use Utopia\Domains\Domain; use Appwrite\Extend\Exception; use Appwrite\Utopia\Database\Validator\Queries\Projects; use Utopia\Cache\Cache; -use Utopia\Database\Validator\Authorization; use Utopia\Pools\Group; use Utopia\Validator\ArrayList; use Utopia\Validator\Boolean; @@ -252,117 +251,6 @@ App::get('/v1/projects/:projectId') $response->dynamic($project, Response::MODEL_PROJECT); }); -App::get('/v1/projects/:projectId/usage') - ->desc('Get usage stats for a project') - ->groups(['api', 'projects']) - ->label('scope', 'projects.read') - ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) - ->label('sdk.namespace', 'projects') - ->label('sdk.method', 'getUsage') - ->label('sdk.response.code', Response::STATUS_CODE_OK) - ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) - ->label('sdk.response.model', Response::MODEL_USAGE_PROJECT) - ->param('projectId', '', new UID(), 'Project unique ID.') - ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true) - ->inject('response') - ->inject('dbForConsole') - ->inject('dbForProject') - ->action(function (string $projectId, string $range, Response $response, Database $dbForConsole, Database $dbForProject) { - - $project = $dbForConsole->getDocument('projects', $projectId); - - if ($project->isEmpty()) { - throw new Exception(Exception::PROJECT_NOT_FOUND); - } - - $usage = []; - if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { - $periods = [ - '24h' => [ - 'period' => '1h', - 'limit' => 24, - ], - '7d' => [ - 'period' => '1d', - 'limit' => 7, - ], - '30d' => [ - 'period' => '1d', - 'limit' => 30, - ], - '90d' => [ - 'period' => '1d', - 'limit' => 90, - ], - ]; - - $dbForProject->setNamespace("_{$project->getInternalId()}"); - - $metrics = [ - 'project.$all.network.requests', - 'project.$all.network.bandwidth', - 'project.$all.storage.size', - 'users.$all.count.total', - 'collections.$all.count.total', - 'documents.$all.count.total', - 'executions.$all.compute.total', - ]; - - $stats = []; - - Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) { - foreach ($metrics as $metric) { - $limit = $periods[$range]['limit']; - $period = $periods[$range]['period']; - - $requestDocs = $dbForProject->find('stats', [ - Query::equal('period', [$period]), - Query::equal('metric', [$metric]), - Query::limit($limit), - Query::orderDesc('time'), - ]); - - $stats[$metric] = []; - foreach ($requestDocs as $requestDoc) { - $stats[$metric][] = [ - 'value' => $requestDoc->getAttribute('value'), - 'date' => $requestDoc->getAttribute('time'), - ]; - } - - // backfill metrics with empty values for graphs - $backfill = $limit - \count($requestDocs); - while ($backfill > 0) { - $last = $limit - $backfill - 1; // array index of last added metric - $diff = match ($period) { // convert period to seconds for unix timestamp math - '1h' => 3600, - '1d' => 86400, - }; - $stats[$metric][] = [ - 'value' => 0, - 'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)), - ]; - $backfill--; - } - $stats[$metric] = array_reverse($stats[$metric]); - } - }); - - $usage = new Document([ - 'range' => $range, - 'requests' => $stats[$metrics[0]] ?? [], - 'network' => $stats[$metrics[1]] ?? [], - 'storage' => $stats[$metrics[2]] ?? [], - 'users' => $stats[$metrics[3]] ?? [], - 'collections' => $stats[$metrics[4]] ?? [], - 'documents' => $stats[$metrics[5]] ?? [], - 'executions' => $stats[$metrics[6]] ?? [], - ]); - } - - $response->dynamic($usage, Response::MODEL_USAGE_PROJECT); - }); - App::patch('/v1/projects/:projectId') ->desc('Update Project') ->groups(['api', 'projects']) diff --git a/composer.json b/composer.json index b2d0d9e846..1d6d1be25b 100644 --- a/composer.json +++ b/composer.json @@ -50,17 +50,17 @@ "utopia-php/cli": "0.14.*", "utopia-php/config": "0.2.*", "utopia-php/database": "0.28.*", - "utopia-php/domains": "1.1.*", - "utopia-php/framework": "0.25.*", - "utopia-php/image": "0.5.*", "utopia-php/queue": "0.5.*", - "utopia-php/locale": "0.4.*", - "utopia-php/logger": "0.3.*", "utopia-php/orchestration": "0.9.*", "utopia-php/platform": "0.3.*", "utopia-php/pools": "0.4.*", "utopia-php/preloader": "0.2.*", + "utopia-php/domains": "1.1.*", + "utopia-php/framework": "0.25.*", + "utopia-php/image": "0.5.*", "utopia-php/dsn": "0.1.*", + "utopia-php/locale": "0.4.*", + "utopia-php/logger": "0.3.*", "utopia-php/messaging": "0.1.*", "utopia-php/registry": "0.5.*", "utopia-php/storage": "0.13.*", diff --git a/composer.lock b/composer.lock index dfeff542b7..7e3705f123 100644 --- a/composer.lock +++ b/composer.lock @@ -2356,16 +2356,16 @@ }, { "name": "utopia-php/pools", - "version": "0.4.1", + "version": "0.4.2", "source": { "type": "git", "url": "https://github.com/utopia-php/pools.git", - "reference": "c8f96a33e7fbf58c1145eb6cf0f2c00cbe319979" + "reference": "d2870ab74b31b7f4027799f082e85122154f8bed" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/pools/zipball/c8f96a33e7fbf58c1145eb6cf0f2c00cbe319979", - "reference": "c8f96a33e7fbf58c1145eb6cf0f2c00cbe319979", + "url": "https://api.github.com/repos/utopia-php/pools/zipball/d2870ab74b31b7f4027799f082e85122154f8bed", + "reference": "d2870ab74b31b7f4027799f082e85122154f8bed", "shasum": "" }, "require": { @@ -2401,9 +2401,9 @@ ], "support": { "issues": "https://github.com/utopia-php/pools/issues", - "source": "https://github.com/utopia-php/pools/tree/0.4.1" + "source": "https://github.com/utopia-php/pools/tree/0.4.2" }, - "time": "2022-11-15T08:55:16+00:00" + "time": "2022-11-22T07:55:45+00:00" }, { "name": "utopia-php/preloader", @@ -5403,5 +5403,5 @@ "platform-overrides": { "php": "8.0" }, - "plugin-api-version": "2.2.0" + "plugin-api-version": "2.3.0" } From dd8a64b3a337cde367a88ca3febef2e23eb62e05 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Tue, 22 Nov 2022 14:10:41 +0000 Subject: [PATCH 19/25] feat: update branch --- app/views/install/compose.phtml | 31 --------------------------- src/Appwrite/Platform/Tasks/Usage.php | 2 +- 2 files changed, 1 insertion(+), 32 deletions(-) diff --git a/app/views/install/compose.phtml b/app/views/install/compose.phtml index 31c5838a1a..37937cfa13 100644 --- a/app/views/install/compose.phtml +++ b/app/views/install/compose.phtml @@ -494,37 +494,6 @@ services: - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG - appwrite-usage-database: - image: /: - entrypoint: - - usage - - --type=database - container_name: appwrite-usage-database - <<: *x-logging - restart: unless-stopped - networks: - - appwrite - depends_on: - - influxdb - - mariadb - environment: - - _APP_ENV - - _APP_OPENSSL_KEY_V1 - - _APP_DB_HOST - - _APP_DB_PORT - - _APP_DB_SCHEMA - - _APP_DB_USER - - _APP_DB_PASS - - _APP_REDIS_HOST - - _APP_REDIS_PORT - - _APP_REDIS_USER - - _APP_REDIS_PASS - - _APP_INFLUXDB_HOST - - _APP_INFLUXDB_PORT - - _APP_USAGE_TIMESERIES_INTERVAL - - _APP_LOGGING_PROVIDER - - _APP_LOGGING_CONFIG - appwrite-schedule: image: /: entrypoint: schedule diff --git a/src/Appwrite/Platform/Tasks/Usage.php b/src/Appwrite/Platform/Tasks/Usage.php index 152523fd37..fa677ea142 100644 --- a/src/Appwrite/Platform/Tasks/Usage.php +++ b/src/Appwrite/Platform/Tasks/Usage.php @@ -41,7 +41,7 @@ class Usage extends Action $errorLogger = fn(Throwable $error, string $action = 'syncUsageStats') => $logError($error, "usage", $action); - $interval = (int) App::getEnv('_APP_USAGE_TIMESERIES_INTERVAL', '30'); // 30 seconds (by default) + $interval = (int) App::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '30'); // 30 seconds (by default) $region = App::getEnv('region', 'default'); $usage = new TimeSeries($region, $dbForConsole, $influxDB, $getProjectDB, $register, $errorLogger); From a47618e01a05f00d6a6e32437c2d5c483d512bd5 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Tue, 22 Nov 2022 14:17:18 +0000 Subject: [PATCH 20/25] feat: update branch --- .env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env b/.env index 9cac2e6438..401f2e9ce8 100644 --- a/.env +++ b/.env @@ -65,7 +65,7 @@ _APP_MAINTENANCE_RETENTION_AUDIT=1209600 _APP_MAINTENANCE_RETENTION_SCHEDULES=86400 _APP_MAINTENANCE_RETENTION_USAGE_HOURLY=8640000 _APP_USAGE_STATS=enabled -_APP_USAGE_AGGREGATION_INTERVAL=5 +_APP_USAGE_AGGREGATION_INTERVAL=30 _APP_LOGGING_PROVIDER= _APP_LOGGING_CONFIG= _APP_REGION=default From 85de5e9106f5af9c516958efec2d1d498815f51d Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Tue, 22 Nov 2022 20:14:39 +0530 Subject: [PATCH 21/25] feat: update droplet config --- .env | 2 +- app/console | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.env b/.env index b7de838c7f..6b31bc1620 100644 --- a/.env +++ b/.env @@ -24,7 +24,7 @@ _APP_DB_SCHEMA=appwrite _APP_DB_USER=user _APP_DB_PASS=password _APP_DB_ROOT_PASS=rootsecretpassword -_APP_CONNECTIONS_MAX=151 +_APP_CONNECTIONS_MAX=300 _APP_POOL_CLIENTS=14 _APP_CONNECTIONS_DB_PROJECT=db_fra1_02=mariadb://user:password@mariadb:3306/appwrite _APP_CONNECTIONS_DB_CONSOLE=db_fra1_01=mariadb://user:password@mariadb:3306/appwrite diff --git a/app/console b/app/console index 25d281795a..4e2cecefb5 160000 --- a/app/console +++ b/app/console @@ -1 +1 @@ -Subproject commit 25d281795a1b73c9b7ed37b14c451068c36d2ff1 +Subproject commit 4e2cecefb571104f0dbbe5a578729f0e17a10242 From 2ef43e3249b8275d5cb20ab371ea1afb4c56e550 Mon Sep 17 00:00:00 2001 From: Matej Baco Date: Tue, 22 Nov 2022 16:36:15 +0100 Subject: [PATCH 22/25] Improvements during QA --- .env | 4 ++-- app/controllers/shared/api.php | 1 - app/workers/functions.php | 5 ++++- docker-compose.yml | 24 ++++++++++++++++++++++++ 4 files changed, 30 insertions(+), 4 deletions(-) diff --git a/.env b/.env index b8bc2d93cf..28161098ad 100644 --- a/.env +++ b/.env @@ -1,6 +1,6 @@ _APP_ENV=development _APP_LOCALE=en -_APP_WORKER_PER_CORE=6 +_APP_WORKER_PER_CORE=2 _APP_CONSOLE_WHITELIST_ROOT=disabled _APP_CONSOLE_WHITELIST_EMAILS= _APP_CONSOLE_WHITELIST_IPS= @@ -24,7 +24,7 @@ _APP_DB_SCHEMA=appwrite _APP_DB_USER=user _APP_DB_PASS=password _APP_DB_ROOT_PASS=rootsecretpassword -_APP_CONNECTIONS_MAX=151 +_APP_CONNECTIONS_MAX=251 _APP_POOL_CLIENTS=14 _APP_CONNECTIONS_DB_PROJECT=db_fra1_02=mariadb://user:password@mariadb:3306/appwrite _APP_CONNECTIONS_DB_CONSOLE=db_fra1_01=mariadb://user:password@mariadb:3306/appwrite diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index db35cebc65..324efa6109 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -192,7 +192,6 @@ App::init() ->setUser($user); $usage - ->setParam('projectInternalId', $project->getInternalId()) ->setParam('projectId', $project->getId()) ->setParam('project.{scope}.network.requests', 1) ->setParam('httpMethod', $request->getMethod()) diff --git a/app/workers/functions.php b/app/workers/functions.php index 3047bf027e..a0c9bbdbac 100644 --- a/app/workers/functions.php +++ b/app/workers/functions.php @@ -152,6 +152,10 @@ Server::setResource('execute', function () { ->setAttribute('status', 'failed') ->setAttribute('statusCode', $th->getCode()) ->setAttribute('stderr', $th->getMessage()); + + Console::error($th->getTraceAsString()); + Console::error($th->getFile()); + Console::error($th->getLine()); Console::error($th->getMessage()); } @@ -204,7 +208,6 @@ Server::setResource('execute', function () { $usage = new Stats($statsd); $usage ->setParam('projectId', $project->getId()) - ->setParam('projectInternalId', $project->getInternalId()) ->setParam('functionId', $function->getId()) ->setParam('executions.{scope}.compute', 1) ->setParam('executionStatus', $execution->getAttribute('status', '')) diff --git a/docker-compose.yml b/docker-compose.yml index 59907aaf04..f0e1636742 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -126,6 +126,7 @@ services: - _APP_DB_USER - _APP_DB_PASS - _APP_CONNECTIONS_MAX + - _APP_POOL_CLIENTS - _APP_REDIS_HOST - _APP_REDIS_PORT - _APP_REDIS_USER @@ -214,6 +215,7 @@ services: - _APP_DB_USER - _APP_DB_PASS - _APP_CONNECTIONS_MAX + - _APP_POOL_CLIENTS - _APP_REDIS_HOST - _APP_REDIS_PORT - _APP_REDIS_USER @@ -277,6 +279,8 @@ services: environment: - _APP_ENV - _APP_WORKER_PER_CORE + - _APP_CONNECTIONS_MAX + - _APP_POOL_CLIENTS - _APP_OPENSSL_KEY_V1 - _APP_SYSTEM_SECURITY_EMAIL_ADDRESS - _APP_REDIS_HOST @@ -308,6 +312,8 @@ services: environment: - _APP_ENV - _APP_WORKER_PER_CORE + - _APP_CONNECTIONS_MAX + - _APP_POOL_CLIENTS - _APP_OPENSSL_KEY_V1 - _APP_DB_HOST - _APP_DB_PORT @@ -344,6 +350,8 @@ services: environment: - _APP_ENV - _APP_WORKER_PER_CORE + - _APP_CONNECTIONS_MAX + - _APP_POOL_CLIENTS - _APP_OPENSSL_KEY_V1 - _APP_DB_HOST - _APP_DB_PORT @@ -377,6 +385,8 @@ services: environment: - _APP_ENV - _APP_WORKER_PER_CORE + - _APP_CONNECTIONS_MAX + - _APP_POOL_CLIENTS - _APP_OPENSSL_KEY_V1 - _APP_EXECUTOR_SECRET - _APP_EXECUTOR_HOST @@ -415,6 +425,8 @@ services: environment: - _APP_ENV - _APP_WORKER_PER_CORE + - _APP_CONNECTIONS_MAX + - _APP_POOL_CLIENTS - _APP_OPENSSL_KEY_V1 - _APP_DOMAIN - _APP_DOMAIN_TARGET @@ -452,6 +464,8 @@ services: environment: - _APP_ENV - _APP_WORKER_PER_CORE + - _APP_CONNECTIONS_MAX + - _APP_POOL_CLIENTS - _APP_OPENSSL_KEY_V1 - _APP_DB_HOST - _APP_DB_PORT @@ -490,6 +504,8 @@ services: environment: - _APP_ENV - _APP_WORKER_PER_CORE + - _APP_CONNECTIONS_MAX + - _APP_POOL_CLIENTS - _APP_OPENSSL_KEY_V1 - _APP_SYSTEM_EMAIL_NAME - _APP_SYSTEM_EMAIL_ADDRESS @@ -521,6 +537,8 @@ services: environment: - _APP_ENV - _APP_WORKER_PER_CORE + - _APP_CONNECTIONS_MAX + - _APP_POOL_CLIENTS - _APP_REDIS_HOST - _APP_REDIS_PORT - _APP_REDIS_USER @@ -546,6 +564,8 @@ services: environment: - _APP_ENV - _APP_WORKER_PER_CORE + - _APP_CONNECTIONS_MAX + - _APP_POOL_CLIENTS - _APP_DOMAIN - _APP_DOMAIN_TARGET - _APP_OPENSSL_KEY_V1 @@ -585,6 +605,8 @@ services: environment: - _APP_ENV - _APP_WORKER_PER_CORE + - _APP_CONNECTIONS_MAX + - _APP_POOL_CLIENTS - _APP_OPENSSL_KEY_V1 - _APP_DB_HOST - _APP_DB_PORT @@ -620,6 +642,8 @@ services: environment: - _APP_ENV - _APP_WORKER_PER_CORE + - _APP_CONNECTIONS_MAX + - _APP_POOL_CLIENTS - _APP_REDIS_HOST - _APP_REDIS_PORT - _APP_REDIS_USER From c7f2785b45560427cb2e2b621c24459adf120be2 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Tue, 22 Nov 2022 21:07:04 +0530 Subject: [PATCH 23/25] feat: update worker counts --- .env | 2 +- app/init.php | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.env b/.env index 6b31bc1620..cbc6ce1472 100644 --- a/.env +++ b/.env @@ -1,6 +1,6 @@ _APP_ENV=development _APP_LOCALE=en -_APP_WORKER_PER_CORE=6 +_APP_WORKER_PER_CORE=3 _APP_CONSOLE_WHITELIST_ROOT=disabled _APP_CONSOLE_WHITELIST_EMAILS= _APP_CONSOLE_WHITELIST_IPS= diff --git a/app/init.php b/app/init.php index 550dc6d744..6af338838d 100644 --- a/app/init.php +++ b/app/init.php @@ -552,10 +552,10 @@ $register->set('pools', function () { ], ]; - $maxConnections = App::getenv('_APP_CONNECTIONS_MAX', 151); - $instanceConnections = $maxConnections / App::getenv('_APP_POOL_CLIENTS', 14); + $maxConnections = App::getEnv('_APP_CONNECTIONS_MAX', 151); + $instanceConnections = $maxConnections / App::getEnv('_APP_POOL_CLIENTS', 14); - $multiprocessing = App::getenv('_APP_SERVER_MULTIPROCESS', 'disabled') === 'enabled'; + $multiprocessing = App::getEnv('_APP_SERVER_MULTIPROCESS', 'disabled') === 'enabled'; if ($multiprocessing) { $workerCount = swoole_cpu_num() * intval(App::getEnv('_APP_WORKER_PER_CORE', 6)); From 95ac8721f0894ca9d27cf8191208cc404b539545 Mon Sep 17 00:00:00 2001 From: Matej Baco Date: Tue, 22 Nov 2022 16:50:41 +0100 Subject: [PATCH 24/25] Revert changes --- app/controllers/shared/api.php | 1 + app/workers/builds.php | 1 + app/workers/functions.php | 3 ++- 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 324efa6109..db35cebc65 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -192,6 +192,7 @@ App::init() ->setUser($user); $usage + ->setParam('projectInternalId', $project->getInternalId()) ->setParam('projectId', $project->getId()) ->setParam('project.{scope}.network.requests', 1) ->setParam('httpMethod', $request->getMethod()) diff --git a/app/workers/builds.php b/app/workers/builds.php index 6b2cfcf66b..d26f07ab75 100644 --- a/app/workers/builds.php +++ b/app/workers/builds.php @@ -254,6 +254,7 @@ class BuildsV1 extends Worker $statsd = $register->get('statsd'); $usage = new Stats($statsd); $usage + ->setParam('projectInternalId', $project->getInternalId()) ->setParam('projectId', $project->getId()) ->setParam('functionId', $function->getId()) ->setParam('builds.{scope}.compute', 1) diff --git a/app/workers/functions.php b/app/workers/functions.php index a0c9bbdbac..2333cb315e 100644 --- a/app/workers/functions.php +++ b/app/workers/functions.php @@ -208,7 +208,8 @@ Server::setResource('execute', function () { $usage = new Stats($statsd); $usage ->setParam('projectId', $project->getId()) - ->setParam('functionId', $function->getId()) + ->setParam('projectInternalId', $project->getInternalId()) + ->setParam('functionId', $function->getId()) // TODO: We should use functionInternalId in usage stats ->setParam('executions.{scope}.compute', 1) ->setParam('executionStatus', $execution->getAttribute('status', '')) ->setParam('executionTime', $execution->getAttribute('duration')) From 92a59cdf3f2d96d8e2762ebb3a924a0d6509e148 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Tue, 22 Nov 2022 21:26:10 +0530 Subject: [PATCH 25/25] fix: use projectInternalId in the usage stats --- src/Appwrite/Usage/Calculators/TimeSeries.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Appwrite/Usage/Calculators/TimeSeries.php b/src/Appwrite/Usage/Calculators/TimeSeries.php index 145b40a402..ce92b6462f 100644 --- a/src/Appwrite/Usage/Calculators/TimeSeries.php +++ b/src/Appwrite/Usage/Calculators/TimeSeries.php @@ -489,7 +489,7 @@ class TimeSeries extends Calculator $query .= "WHERE \"time\" > '{$start}' "; $query .= "AND \"time\" < '{$end}' "; $query .= "AND \"metric_type\"='counter' {$filters} "; - $query .= "GROUP BY time({$period['key']}), \"projectId\" {$groupBy} "; + $query .= "GROUP BY time({$period['key']}), \"projectId\", \"projectInternalId\" {$groupBy} "; $query .= "FILL(null)"; try { @@ -516,7 +516,7 @@ class TimeSeries extends Calculator } $this->createOrUpdateMetric( - $point['projectId'], + $point['projectInternalId'], $point['time'], $period['key'], $metricUpdated,