From 95a9ded28f52a3aaa39fdc278794f6e80110e680 Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Sat, 15 Oct 2022 21:17:03 +0300 Subject: [PATCH] Adjusted compose files and tests --- app/init.php | 7 ++- app/views/install/compose.phtml | 31 ++++++++++-- docker-compose.yml | 21 ++++++++ src/Appwrite/Resque/Worker.php | 87 ++++++++++++++++++++++----------- tests/unit/DSN/DSNTest.php | 4 +- 5 files changed, 110 insertions(+), 40 deletions(-) diff --git a/app/init.php b/app/init.php index d4cd78515..c4f7f3236 100644 --- a/app/init.php +++ b/app/init.php @@ -498,7 +498,6 @@ $register->set('logger', function () { $adapter = new $classname($providerConfig); return new Logger($adapter); }); - $register->set('pools', function () { $group= new Group(); @@ -551,7 +550,8 @@ $register->set('pools', function () { $config[] = $name; if(empty($dsn)) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, "Missing value for DSN connection in {$key}"); + //throw new Exception(Exception::GENERAL_SERVER_ERROR, "Missing value for DSN connection in {$key}"); + continue; } $dsn = new DSN($dsn); @@ -652,7 +652,6 @@ $register->set('pools', function () { return $group; }); - $register->set('influxdb', function () { // Register DB connection $host = App::getEnv('_APP_INFLUXDB_HOST', ''); @@ -1009,7 +1008,7 @@ App::setResource('dbForProject', function (Group $pools, Database $dbForConsole, if($project->isEmpty() || $project->getId() === 'console') { return $dbForConsole; } - + $dbAdapter = $pools ->get($project->getAttribute('database')) ->pop() diff --git a/app/views/install/compose.phtml b/app/views/install/compose.phtml index c10a02750..164e370d9 100644 --- a/app/views/install/compose.phtml +++ b/app/views/install/compose.phtml @@ -94,6 +94,9 @@ services: - _APP_REDIS_PASS - _APP_CONNECTIONS_DB_CONSOLE - _APP_CONNECTIONS_DB_PROJECT + - _APP_CONNECTIONS_CACHE + - _APP_CONNECTIONS_QUEUE + - _APP_CONNECTIONS_PUBSUB - _APP_SMTP_HOST - _APP_SMTP_PORT - _APP_SMTP_SECURE @@ -185,6 +188,8 @@ services: - _APP_REDIS_PORT - _APP_CONNECTIONS_DB_CONSOLE - _APP_CONNECTIONS_DB_PROJECT + - _APP_CONNECTIONS_CACHE + - _APP_CONNECTIONS_PUBSUB - _APP_USAGE_STATS - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG @@ -209,6 +214,8 @@ services: - _APP_REDIS_PASS - _APP_CONNECTIONS_DB_CONSOLE - _APP_CONNECTIONS_DB_PROJECT + - _APP_CONNECTIONS_CACHE + - _APP_CONNECTIONS_QUEUE - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG @@ -231,6 +238,7 @@ services: - _APP_REDIS_PORT - _APP_REDIS_USER - _APP_REDIS_PASS + - _APP_CONNECTIONS_QUEUE - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG @@ -260,6 +268,8 @@ services: - _APP_REDIS_PASS - _APP_CONNECTIONS_DB_CONSOLE - _APP_CONNECTIONS_DB_PROJECT + - _APP_CONNECTIONS_CACHE + - _APP_CONNECTIONS_QUEUE - _APP_STORAGE_DEVICE - _APP_STORAGE_S3_ACCESS_KEY - _APP_STORAGE_S3_SECRET @@ -306,6 +316,8 @@ services: - _APP_REDIS_PASS - _APP_CONNECTIONS_DB_CONSOLE - _APP_CONNECTIONS_DB_PROJECT + - _APP_CONNECTIONS_CACHE + - _APP_CONNECTIONS_QUEUE - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG @@ -331,6 +343,8 @@ services: - _APP_REDIS_PASS - _APP_CONNECTIONS_DB_CONSOLE - _APP_CONNECTIONS_DB_PROJECT + - _APP_CONNECTIONS_CACHE + - _APP_CONNECTIONS_QUEUE - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG @@ -360,6 +374,8 @@ services: - _APP_REDIS_PASS - _APP_CONNECTIONS_DB_CONSOLE - _APP_CONNECTIONS_DB_PROJECT + - _APP_CONNECTIONS_CACHE + - _APP_CONNECTIONS_QUEUE - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG @@ -384,6 +400,8 @@ services: - _APP_REDIS_PASS - _APP_CONNECTIONS_DB_CONSOLE - _APP_CONNECTIONS_DB_PROJECT + - _APP_CONNECTIONS_CACHE + - _APP_CONNECTIONS_QUEUE - _APP_FUNCTIONS_TIMEOUT - _APP_EXECUTOR_SECRET - _APP_EXECUTOR_HOST @@ -468,6 +486,7 @@ services: - _APP_REDIS_PORT - _APP_REDIS_USER - _APP_REDIS_PASS + - _APP_CONNECTIONS_QUEUE - _APP_SMTP_HOST - _APP_SMTP_PORT - _APP_SMTP_SECURE @@ -491,6 +510,7 @@ services: - _APP_REDIS_PORT - _APP_REDIS_USER - _APP_REDIS_PASS + - _APP_CONNECTIONS_QUEUE - _APP_SMS_PROVIDER - _APP_SMS_FROM - _APP_LOGGING_PROVIDER @@ -517,6 +537,7 @@ services: - _APP_REDIS_PASS - _APP_CONNECTIONS_DB_CONSOLE - _APP_CONNECTIONS_DB_PROJECT + - _APP_CONNECTIONS_CACHE - _APP_MAINTENANCE_INTERVAL - _APP_MAINTENANCE_RETENTION_EXECUTION - _APP_MAINTENANCE_RETENTION_CACHE @@ -541,6 +562,7 @@ services: - _APP_OPENSSL_KEY_V1 - _APP_CONNECTIONS_DB_CONSOLE - _APP_CONNECTIONS_DB_PROJECT + - _APP_CONNECTIONS_CACHE - _APP_INFLUXDB_HOST - _APP_INFLUXDB_PORT - _APP_USAGE_TIMESERIES_INTERVAL @@ -568,11 +590,9 @@ services: environment: - _APP_ENV - _APP_OPENSSL_KEY_V1 - - _APP_DB_HOST - - _APP_DB_PORT - - _APP_DB_SCHEMA - - _APP_DB_USER - - _APP_DB_PASS + - _APP_CONNECTIONS_DB_CONSOLE + - _APP_CONNECTIONS_DB_PROJECT + - _APP_CONNECTIONS_CACHE - _APP_INFLUXDB_HOST - _APP_INFLUXDB_PORT - _APP_USAGE_TIMESERIES_INTERVAL @@ -600,6 +620,7 @@ services: - _APP_REDIS_PORT - _APP_REDIS_USER - _APP_REDIS_PASS + - _APP_CONNECTIONS_QUEUE mariadb: image: mariadb:10.7 # fix issues when upgrading using: mysql_upgrade -u root -p diff --git a/docker-compose.yml b/docker-compose.yml index 94446d535..4dff4f5ff 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -217,6 +217,8 @@ services: - _APP_REDIS_PORT - _APP_CONNECTIONS_DB_CONSOLE - _APP_CONNECTIONS_DB_PROJECT + - _APP_CONNECTIONS_CACHE + - _APP_CONNECTIONS_PUBSUB - _APP_USAGE_STATS - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG @@ -244,6 +246,8 @@ services: - _APP_REDIS_PASS - _APP_CONNECTIONS_DB_CONSOLE - _APP_CONNECTIONS_DB_PROJECT + - _APP_CONNECTIONS_CACHE + - _APP_CONNECTIONS_QUEUE - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG @@ -270,6 +274,7 @@ services: - _APP_REDIS_PORT - _APP_REDIS_USER - _APP_REDIS_PASS + - _APP_CONNECTIONS_QUEUE - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG @@ -301,6 +306,8 @@ services: - _APP_REDIS_PASS - _APP_CONNECTIONS_DB_CONSOLE - _APP_CONNECTIONS_DB_PROJECT + - _APP_CONNECTIONS_CACHE + - _APP_CONNECTIONS_QUEUE - *x-env-storage - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG @@ -331,6 +338,8 @@ services: - _APP_REDIS_PASS - _APP_CONNECTIONS_DB_CONSOLE - _APP_CONNECTIONS_DB_PROJECT + - _APP_CONNECTIONS_CACHE + - _APP_CONNECTIONS_QUEUE - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG @@ -359,6 +368,8 @@ services: - _APP_REDIS_PASS - _APP_CONNECTIONS_DB_CONSOLE - _APP_CONNECTIONS_DB_PROJECT + - _APP_CONNECTIONS_CACHE + - _APP_CONNECTIONS_QUEUE - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG @@ -390,6 +401,8 @@ services: - _APP_REDIS_PASS - _APP_CONNECTIONS_DB_CONSOLE - _APP_CONNECTIONS_DB_PROJECT + - _APP_CONNECTIONS_CACHE + - _APP_CONNECTIONS_QUEUE - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG @@ -417,6 +430,8 @@ services: - _APP_REDIS_PASS - _APP_CONNECTIONS_DB_CONSOLE - _APP_CONNECTIONS_DB_PROJECT + - _APP_CONNECTIONS_CACHE + - _APP_CONNECTIONS_QUEUE - _APP_FUNCTIONS_TIMEOUT - _APP_EXECUTOR_SECRET - _APP_EXECUTOR_HOST @@ -494,6 +509,7 @@ services: - _APP_REDIS_PORT - _APP_REDIS_USER - _APP_REDIS_PASS + - _APP_CONNECTIONS_QUEUE - _APP_SMTP_HOST - _APP_SMTP_PORT - _APP_SMTP_SECURE @@ -521,6 +537,7 @@ services: - _APP_REDIS_PORT - _APP_REDIS_USER - _APP_REDIS_PASS + - _APP_CONNECTIONS_QUEUE - _APP_SMS_PROVIDER - _APP_SMS_FROM - _APP_LOGGING_PROVIDER @@ -551,6 +568,7 @@ services: - _APP_REDIS_PASS - _APP_CONNECTIONS_DB_CONSOLE - _APP_CONNECTIONS_DB_PROJECT + - _APP_CONNECTIONS_CACHE - _APP_MAINTENANCE_INTERVAL - _APP_MAINTENANCE_RETENTION_EXECUTION - _APP_MAINTENANCE_RETENTION_CACHE @@ -581,6 +599,7 @@ services: - _APP_OPENSSL_KEY_V1 - _APP_CONNECTIONS_DB_CONSOLE - _APP_CONNECTIONS_DB_PROJECT + - _APP_CONNECTIONS_CACHE - _APP_INFLUXDB_HOST - _APP_INFLUXDB_PORT - _APP_USAGE_TIMESERIES_INTERVAL @@ -616,6 +635,7 @@ services: - _APP_OPENSSL_KEY_V1 - _APP_CONNECTIONS_DB_CONSOLE - _APP_CONNECTIONS_DB_PROJECT + - _APP_CONNECTIONS_CACHE - _APP_INFLUXDB_HOST - _APP_INFLUXDB_PORT - _APP_USAGE_TIMESERIES_INTERVAL @@ -646,6 +666,7 @@ services: - _APP_REDIS_PORT - _APP_REDIS_USER - _APP_REDIS_PASS + - _APP_CONNECTIONS_QUEUE mariadb: image: mariadb:10.7 # fix issues when upgrading using: mysql_upgrade -u root -p diff --git a/src/Appwrite/Resque/Worker.php b/src/Appwrite/Resque/Worker.php index e394f416f..0010f491e 100644 --- a/src/Appwrite/Resque/Worker.php +++ b/src/Appwrite/Resque/Worker.php @@ -5,6 +5,8 @@ namespace Appwrite\Resque; use Exception; use Appwrite\Database\Pools; use Utopia\App; +use Utopia\Cache\Adapter\None; +use Utopia\Cache\Cache; use Utopia\Database\Database; use Utopia\Storage\Device; use Utopia\Storage\Storage; @@ -134,8 +136,14 @@ abstract class Worker */ public function tearDown(): void { + global $register; + try { + $pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */ + $pools->reclaim(); + $this->shutdown(); + } catch (\Throwable $error) { foreach (self::$errorCallbacks as $errorCallback) { $errorCallback($error, "shutdown", $this->getName()); @@ -165,22 +173,24 @@ abstract class Worker protected function getProjectDB(Document $project): Database { global $register; - $database = $project->getAttribute('database', ''); - $internalId = $project->getInternalId(); - if (empty($database)) { - throw new \Exception('Database name not provided - cannot get database'); + + $pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */ + + if($project->isEmpty() || $project->getId() === 'console') { + return $this->getConsoleDB(); } - - $cache = $register->get('cache'); - $dbPool = $register->get('dbPool'); - $namespace = "_$internalId"; - $pdo = $dbPool->getPDO($database); - $dbForProject = Pools::wait( - Pools::getDatabase($pdo, $cache, $namespace), - 'projects' - ); - - return $dbForProject; + + $dbAdapter = $pools + ->get($project->getAttribute('database')) + ->pop() + ->getResource() + ; + + $database = new Database($dbAdapter, $this->getCache()); + $database->setNamespace('_'.$project->getInternalId()); + $database->setDefaultDatabase('appwrite'); + + return $database; } /** @@ -190,21 +200,41 @@ abstract class Worker protected function getConsoleDB(): Database { global $register; - $cache = $register->get('cache'); - $dbPool = $register->get('dbPool'); - $database = $dbPool->getConsoleDB(); - if (empty($database)) { - throw new \Exception('Database name not provided - cannot get database'); - } - $namespace = "_console"; - $pdo = $dbPool->getPDO($database); - $dbForConsole = Pools::wait( - Pools::getDatabase($pdo, $cache, $namespace), - '_metadata' - ); + $pools = $register->get('pools'); /* @var \Utopia\Pools\Group $pools */ + + $dbAdapter = $pools + ->get('console') + ->pop() + ->getResource() + ; - return $dbForConsole; + $database = new Database($dbAdapter, $this->getCache()); + + $database->setNamespace('console'); + $database->setDefaultDatabase('appwrite'); + + return $database; + } + + /** + * Get Cache + * @return Cache + */ + protected function getCache(): Cache + { + global $register; + + $pools = $register->get('pools'); /* @var \Utopia\Pools\Group $pools */ + + $pools + ->get('cache') + ->pop() + ->getResource() + ; + + return new Cache(new None()); + // return new Cache($cacheAdapter); } /** @@ -227,7 +257,6 @@ abstract class Worker return $this->getDevice(APP_STORAGE_UPLOADS . '/app-' . $projectId); } - /** * Get Builds Storage Device * @param string $projectId of the project diff --git a/tests/unit/DSN/DSNTest.php b/tests/unit/DSN/DSNTest.php index d1f5ba119..6565e0da1 100644 --- a/tests/unit/DSN/DSNTest.php +++ b/tests/unit/DSN/DSNTest.php @@ -14,7 +14,7 @@ class DSNTest extends TestCase $this->assertEquals("user", $dsn->getUser()); $this->assertEquals("password", $dsn->getPassword()); $this->assertEquals("localhost", $dsn->getHost()); - $this->assertEquals("3306", $dsn->getPort()); + $this->assertEquals(3306, $dsn->getPort()); $this->assertEquals("database", $dsn->getDatabase()); $this->assertEquals("charset=utf8&timezone=UTC", $dsn->getQuery()); @@ -23,7 +23,7 @@ class DSNTest extends TestCase $this->assertEquals("user", $dsn->getUser()); $this->assertNull($dsn->getPassword()); $this->assertEquals("localhost", $dsn->getHost()); - $this->assertEquals("3306", $dsn->getPort()); + $this->assertEquals(3306, $dsn->getPort()); $this->assertEquals("database", $dsn->getDatabase()); $this->assertEquals("charset=utf8&timezone=UTC", $dsn->getQuery());