diff --git a/.env b/.env index 18df078881..950fd44821 100644 --- a/.env +++ b/.env @@ -23,8 +23,11 @@ _APP_DB_SCHEMA=appwrite _APP_DB_USER=user _APP_DB_PASS=password _APP_DB_ROOT_PASS=rootsecretpassword -_APP_DB_PROJECT=db_fra1_02=mysql://user:password@mariadb:3306/appwrite -_APP_DB_CONSOLE=db_fra1_01=mysql://user:password@mariadb:3306/appwrite +_APP_CONNECTIONS_DB_PROJECT=db_fra1_02=mysql://user:password@mariadb:3306/appwrite +_APP_CONNECTIONS_DB_CONSOLE=db_fra1_01=mysql://user:password@mariadb:3306/appwrite +_APP_CONNECTIONS_CACHE=redis_fra1_01=redis://redis:6379 +_APP_CONNECTIONS_QUEUE=redis_fra1_01=redis://redis:6379 +_APP_CONNECTIONS_PUBSUB=redis_fra1_01=redis://redis:6379 _APP_STORAGE_DEVICE=Local _APP_STORAGE_S3_ACCESS_KEY= _APP_STORAGE_S3_SECRET= diff --git a/Dockerfile b/Dockerfile index a7cae38502..410fb1c44c 100755 --- a/Dockerfile +++ b/Dockerfile @@ -35,6 +35,7 @@ ENV PHP_REDIS_VERSION=5.3.7 \ PHP_IMAGICK_VERSION=3.7.0 \ PHP_YAML_VERSION=2.2.2 \ PHP_MAXMINDDB_VERSION=v1.11.0 \ + PHP_MEMCACHED_VERSION=v3.2.0 \ PHP_ZSTD_VERSION="4504e4186e79b197cfcb75d4d09aa47ef7d92fe9 " RUN \ @@ -52,6 +53,7 @@ RUN \ imagemagick \ imagemagick-dev \ libmaxminddb-dev \ + libmemcached-dev \ zstd-dev RUN docker-php-ext-install sockets @@ -125,6 +127,15 @@ RUN \ ./configure && \ make && make install +# Memcached Extension +FROM compile as memcached +RUN \ + git clone --depth 1 --branch $PHP_MEMCACHED_VERSION https://github.com/php-memcached-dev/php-memcached.git && \ + cd php-memcached && \ + phpize && \ + ./configure && \ + make && make install + # Zstd Compression FROM compile as zstd RUN git clone --recursive -n https://github.com/kjdev/php-ext-zstd.git \ @@ -134,7 +145,6 @@ RUN git clone --recursive -n https://github.com/kjdev/php-ext-zstd.git \ && ./configure --with-libzstd \ && make && make install - # Rust Extensions Compile Image FROM php:8.0.18-cli as rust_compile @@ -304,6 +314,7 @@ COPY --from=imagick /usr/local/lib/php/extensions/no-debug-non-zts-20200930/imag COPY --from=yaml /usr/local/lib/php/extensions/no-debug-non-zts-20200930/yaml.so /usr/local/lib/php/extensions/no-debug-non-zts-20200930/ COPY --from=maxmind /usr/local/lib/php/extensions/no-debug-non-zts-20200930/maxminddb.so /usr/local/lib/php/extensions/no-debug-non-zts-20200930/ COPY --from=mongodb /usr/local/lib/php/extensions/no-debug-non-zts-20200930/mongodb.so /usr/local/lib/php/extensions/no-debug-non-zts-20200930/ +COPY --from=memcached /usr/local/lib/php/extensions/no-debug-non-zts-20200930/memcached.so /usr/local/lib/php/extensions/no-debug-non-zts-20200930/ COPY --from=scrypt /usr/local/lib/php/extensions/php-scrypt/target/libphp_scrypt.so /usr/local/lib/php/extensions/no-debug-non-zts-20200930/ COPY --from=zstd /usr/local/lib/php/extensions/no-debug-non-zts-20200930/zstd.so /usr/local/lib/php/extensions/no-debug-non-zts-20200930/ diff --git a/app/cli.php b/app/cli.php index 09b3dc9413..3a62c80816 100644 --- a/app/cli.php +++ b/app/cli.php @@ -6,7 +6,78 @@ require_once __DIR__ . '/controllers/general.php'; use Utopia\App; use Utopia\CLI\CLI; use Utopia\CLI\Console; +use Utopia\Cache\Adapter\Sharding; +use Utopia\Cache\Cache; +use Utopia\Config\Config; +use Utopia\Database\Database; use Utopia\Database\Validator\Authorization; +use InfluxDB\Database as InfluxDatabase; + +function getInfluxDB(): InfluxDatabase +{ + global $register; + + $client = $register->get('influxdb'); /** @var InfluxDB\Client $client */ + $attempts = 0; + $max = 10; + $sleep = 1; + + do { // check if telegraf database is ready + try { + $attempts++; + $database = $client->selectDB('telegraf'); + if (in_array('telegraf', $client->listDatabases())) { + break; // leave the do-while if successful + } + } catch (\Throwable $th) { + Console::warning("InfluxDB not ready. Retrying connection ({$attempts})..."); + if ($attempts >= $max) { + throw new \Exception('InfluxDB database not ready yet'); + } + sleep($sleep); + } + } while ($attempts < $max); + return $database; +} + +function getConsoleDB(): Database +{ + global $register; + + $pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */ + + $dbAdapter = $pools + ->get('console') + ->pop() + ->getResource() + ; + + $database = new Database($dbAdapter, getCache()); + + $database->setNamespace('console'); + + return $database; +} + +function getCache(): Cache +{ + global $register; + + $pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */ + + $list = Config::getParam('pools-cache', []); + $adapters = []; + + foreach ($list as $value) { + $adapters[] = $pools + ->get($value) + ->pop() + ->getResource() + ; + } + + return new Cache(new Sharding($adapters)); +} Authorization::disable(); @@ -29,4 +100,13 @@ $cli Console::log(App::getEnv('_APP_VERSION', 'UNKNOWN')); }); +$cli + ->error(function ($error) { + if (App::getEnv('_APP_ENV', 'development')) { + Console::error($error); + } else { + Console::error($error->getMessage()); + } + }); + $cli->run(); diff --git a/app/config/variables.php b/app/config/variables.php index 6bcf4097ed..f529831192 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -306,24 +306,24 @@ return [ 'question' => '', 'filter' => 'password' ], - [ - 'name' => '_APP_DB_PROJECT', - 'description' => 'A list of comma-separated key value pairs representing Project DBs where key is the database name and value is the DSN connection string.', - 'introduction' => 'TBD', - 'default' => 'db_fra1_01=mysql://user:password@mariadb:3306/appwrite', - 'required' => true, - 'question' => '', - 'filter' => '' - ], - [ - 'name' => '_APP_DB__APP_DB_CONSOLEROOT_PASS', - 'description' => 'A key value pair representing the Console DB where key is the database name and value is the DSN connection string.', - 'introduction' => 'TBD', - 'default' => 'db_fra1_01=mysql://user:password@mariadb:3306/appwrite', - 'required' => true, - 'question' => '', - 'filter' => '' - ] + // [ + // 'name' => '_APP_CONNECTIONS_DB_PROJECT', + // 'description' => 'A list of comma-separated key value pairs representing Project DBs where key is the database name and value is the DSN connection string.', + // 'introduction' => 'TBD', + // 'default' => 'db_fra1_01=mysql://user:password@mariadb:3306/appwrite', + // 'required' => true, + // 'question' => '', + // 'filter' => '' + // ], + // [ + // 'name' => '_APP_CONNECTIONS_DB_CONSOLE', + // 'description' => 'A key value pair representing the Console DB where key is the database name and value is the DSN connection string.', + // 'introduction' => 'TBD', + // 'default' => 'db_fra1_01=mysql://user:password@mariadb:3306/appwrite', + // 'required' => true, + // 'question' => '', + // 'filter' => '' + // ] ], ], [ diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index fb91a4517d..8bebf307b9 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -5,7 +5,6 @@ use Appwrite\Auth\Auth; use Appwrite\Auth\Validator\Password; use Appwrite\Auth\Validator\Phone; use Appwrite\Detector\Detector; -use Appwrite\Event\Audit; use Appwrite\Event\Event; use Appwrite\Event\Mail; use Appwrite\Event\Phone as EventPhone; @@ -39,7 +38,6 @@ use Utopia\Database\Validator\UID; use Utopia\Locale\Locale; use Utopia\Validator\ArrayList; use Utopia\Validator\Assoc; -use Utopia\Validator\Range; use Utopia\Validator\Text; use Utopia\Validator\WhiteList; diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index d3a26b414e..f2745281fc 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -25,7 +25,6 @@ use Appwrite\Task\Validator\Cron; use Appwrite\Utopia\Database\Validator\Queries\Deployments; use Appwrite\Utopia\Database\Validator\Queries\Executions; use Appwrite\Utopia\Database\Validator\Queries\Functions; -use Appwrite\Utopia\Database\Validator\Queries\Variables; use Utopia\App; use Utopia\Database\Database; use Utopia\Database\Document; @@ -33,7 +32,6 @@ use Utopia\Database\DateTime; use Utopia\Database\Query; use Utopia\Database\Validator\Authorization; use Utopia\Validator\ArrayList; -use Utopia\Validator\Assoc; use Utopia\Validator\Text; use Utopia\Validator\Range; use Utopia\Validator\WhiteList; diff --git a/app/controllers/api/health.php b/app/controllers/api/health.php index 4842d3f528..f65e65ba23 100644 --- a/app/controllers/api/health.php +++ b/app/controllers/api/health.php @@ -5,7 +5,9 @@ use Appwrite\Event\Event; use Appwrite\Extend\Exception; use Appwrite\Utopia\Response; use Utopia\App; +use Utopia\Config\Config; use Utopia\Database\Document; +use Utopia\Pools\Group; use Utopia\Registry\Registry; use Utopia\Storage\Device; use Utopia\Storage\Device\Local; @@ -26,6 +28,7 @@ App::get('/v1/health') ->action(function (Response $response) { $output = [ + 'name' => 'http', 'status' => 'pass', 'ping' => 0 ]; @@ -42,7 +45,6 @@ App::get('/v1/health/version') ->label('sdk.response.model', Response::MODEL_HEALTH_VERSION) ->inject('response') ->action(function (Response $response) { - $response->dynamic(new Document([ 'version' => APP_VERSION_STABLE ]), Response::MODEL_HEALTH_VERSION); }); @@ -58,33 +60,50 @@ App::get('/v1/health/db') ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_HEALTH_STATUS) ->inject('response') - ->inject('utopia') - ->action(function (Response $response, App $utopia) { + ->inject('pools') + ->action(function (Response $response, Group $pools) { - $checkStart = \microtime(true); + $output = []; - try { - $dbPool = $utopia->getResource('dbPool'); - $database = $dbPool->getConsoleDB(); - /* @var $consoleDB PDO */ - $consoleDB = $dbPool->getPDO($database); - - // Run a small test to check the connection - $statement = $consoleDB->prepare("SELECT 1;"); - - $statement->closeCursor(); - - $statement->execute(); - } catch (Exception $_e) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Database is not available'); - } - - $output = [ - 'status' => 'pass', - 'ping' => \round((\microtime(true) - $checkStart) / 1000) + $configs = [ + 'Console.DB' => Config::getParam('pools-console'), + 'Projects.DB' => Config::getParam('pools-database'), ]; - $response->dynamic(new Document($output), Response::MODEL_HEALTH_STATUS); + foreach ($configs as $key => $config) { + foreach ($config as $database) { + try { + $adapter = $pools->get($database)->pop()->getResource(); + + $checkStart = \microtime(true); + + if ($adapter->ping()) { + $output[] = new Document([ + 'name' => $key . " ($database)", + 'status' => 'pass', + 'ping' => \round((\microtime(true) - $checkStart) / 1000) + ]); + } else { + $output[] = new Document([ + 'name' => $key . " ($database)", + 'status' => 'fail', + 'ping' => \round((\microtime(true) - $checkStart) / 1000) + ]); + } + } catch (\Throwable $th) { + $output[] = new Document([ + 'name' => $key . " ($database)", + 'status' => 'fail', + 'ping' => \round((\microtime(true) - $checkStart) / 1000) + ]); + } + } + } + + $response->dynamic(new Document([ + 'statuses' => $output, + 'total' => count($output), + ]), Response::MODEL_HEALTH_STATUS_LIST); }); App::get('/v1/health/cache') @@ -99,23 +118,163 @@ App::get('/v1/health/cache') ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_HEALTH_STATUS) ->inject('response') - ->inject('utopia') - ->action(function (Response $response, App $utopia) { + ->inject('pools') + ->action(function (Response $response, Group $pools) { - $checkStart = \microtime(true); + $output = []; - $redis = $utopia->getResource('cache'); - - if (!$redis->ping(true)) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Cache is not available'); - } - - $output = [ - 'status' => 'pass', - 'ping' => \round((\microtime(true) - $checkStart) / 1000) + $configs = [ + 'Cache' => Config::getParam('pools-cache'), ]; - $response->dynamic(new Document($output), Response::MODEL_HEALTH_STATUS); + foreach ($configs as $key => $config) { + foreach ($config as $database) { + try { + $adapter = $pools->get($database)->pop()->getResource(); + + $checkStart = \microtime(true); + + if ($adapter->ping()) { + $output[] = new Document([ + 'name' => $key . " ($database)", + 'status' => 'pass', + 'ping' => \round((\microtime(true) - $checkStart) / 1000) + ]); + } else { + $output[] = new Document([ + 'name' => $key . " ($database)", + 'status' => 'fail', + 'ping' => \round((\microtime(true) - $checkStart) / 1000) + ]); + } + } catch (\Throwable $th) { + $output[] = new Document([ + 'name' => $key . " ($database)", + 'status' => 'fail', + 'ping' => \round((\microtime(true) - $checkStart) / 1000) + ]); + } + } + } + + $response->dynamic(new Document([ + 'statuses' => $output, + 'total' => count($output), + ]), Response::MODEL_HEALTH_STATUS_LIST); + }); + +App::get('/v1/health/queue') + ->desc('Get Queue') + ->groups(['api', 'health']) + ->label('scope', 'health.read') + ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) + ->label('sdk.namespace', 'health') + ->label('sdk.method', 'getQueue') + ->label('sdk.description', '/docs/references/health/get-queue.md') + ->label('sdk.response.code', Response::STATUS_CODE_OK) + ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) + ->label('sdk.response.model', Response::MODEL_HEALTH_STATUS) + ->inject('response') + ->inject('pools') + ->action(function (Response $response, Group $pools) { + + $output = []; + + $configs = [ + 'Queue' => Config::getParam('pools-queue'), + ]; + + foreach ($configs as $key => $config) { + foreach ($config as $database) { + try { + $adapter = $pools->get($database)->pop()->getResource(); + + $checkStart = \microtime(true); + + if ($adapter->ping()) { + $output[] = new Document([ + 'name' => $key . " ($database)", + 'status' => 'pass', + 'ping' => \round((\microtime(true) - $checkStart) / 1000) + ]); + } else { + $output[] = new Document([ + 'name' => $key . " ($database)", + 'status' => 'fail', + 'ping' => \round((\microtime(true) - $checkStart) / 1000) + ]); + } + } catch (\Throwable $th) { + $output[] = new Document([ + 'name' => $key . " ($database)", + 'status' => 'fail', + 'ping' => \round((\microtime(true) - $checkStart) / 1000) + ]); + } + } + } + + $response->dynamic(new Document([ + 'statuses' => $output, + 'total' => count($output), + ]), Response::MODEL_HEALTH_STATUS_LIST); + }); + +App::get('/v1/health/pubsub') + ->desc('Get PubSub') + ->groups(['api', 'health']) + ->label('scope', 'health.read') + ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) + ->label('sdk.namespace', 'health') + ->label('sdk.method', 'getPubSub') + ->label('sdk.description', '/docs/references/health/get-pubsub.md') + ->label('sdk.response.code', Response::STATUS_CODE_OK) + ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) + ->label('sdk.response.model', Response::MODEL_HEALTH_STATUS) + ->inject('response') + ->inject('pools') + ->action(function (Response $response, Group $pools) { + + $output = []; + + $configs = [ + 'PubSub' => Config::getParam('pools-pubsub'), + ]; + + foreach ($configs as $key => $config) { + foreach ($config as $database) { + try { + $adapter = $pools->get($database)->pop()->getResource(); + + $checkStart = \microtime(true); + + if ($adapter->ping()) { + $output[] = new Document([ + 'name' => $key . " ($database)", + 'status' => 'pass', + 'ping' => \round((\microtime(true) - $checkStart) / 1000) + ]); + } else { + $output[] = new Document([ + 'name' => $key . " ($database)", + 'status' => 'fail', + 'ping' => \round((\microtime(true) - $checkStart) / 1000) + ]); + } + } catch (\Throwable $th) { + $output[] = new Document([ + 'name' => $key . " ($database)", + 'status' => 'fail', + 'ping' => \round((\microtime(true) - $checkStart) / 1000) + ]); + } + } + } + + $response->dynamic(new Document([ + 'statuses' => $output, + 'total' => count($output), + ]), Response::MODEL_HEALTH_STATUS_LIST); }); App::get('/v1/health/time') diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index a79daf533c..74b9cc298e 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -2,7 +2,6 @@ use Appwrite\Auth\Auth; use Appwrite\Auth\Validator\Password; -use Appwrite\Database\DatabasePool; use Appwrite\Event\Certificate; use Appwrite\Event\Delete; use Appwrite\Event\Validator\Event; @@ -29,10 +28,9 @@ use Utopia\Database\Validator\UID; use Utopia\Domains\Domain; use Utopia\Registry\Registry; use Appwrite\Extend\Exception; -use Utopia\Cache\Adapter\Redis; -use Utopia\Cache\Cache; -use Utopia\Database\Adapter\MariaDB; use Appwrite\Utopia\Database\Validator\Queries\Projects; +use Utopia\Cache\Cache; +use Utopia\Pools\Group; use Utopia\Validator\ArrayList; use Utopia\Validator\Boolean; use Utopia\Validator\Hostname; @@ -74,8 +72,9 @@ App::post('/v1/projects') ->inject('response') ->inject('dbForConsole') ->inject('cache') - ->inject('dbPool') - ->action(function (string $projectId, string $name, string $teamId, string $region, string $description, string $logo, string $url, string $legalName, string $legalCountry, string $legalState, string $legalCity, string $legalAddress, string $legalTaxId, Response $response, Database $dbForConsole, \Redis $cache, DatabasePool $dbPool) { + ->inject('pools') + ->action(function (string $projectId, string $name, string $teamId, string $description, string $logo, string $url, string $legalName, string $legalCountry, string $legalState, string $legalCity, string $legalAddress, string $legalTaxId, Response $response, Database $dbForConsole, Cache $cache, Group $pools) { + $team = $dbForConsole->getDocument('teams', $teamId); @@ -90,13 +89,13 @@ App::post('/v1/projects') } $projectId = ($projectId == 'unique()') ? ID::unique() : $projectId; + $databases = Config::getParam('pools-database', []); + $database = $databases[array_rand($databases)]; if ($projectId === 'console') { throw new Exception(Exception::PROJECT_RESERVED_PROJECT, "'console' is a reserved project."); } - $pdo = $dbPool->getAnyFromPool(); - $project = $dbForConsole->createDocument('projects', new Document([ '$id' => $projectId, '$permissions' => [ @@ -127,10 +126,12 @@ App::post('/v1/projects') 'domains' => null, 'auths' => $auths, 'search' => implode(' ', [$projectId, $name]), - 'database' => $pdo->getName() + 'database' => $database, ])); - $dbForProject = DatabasePool::getDatabase($pdo->getConnection(), $cache, "_{$project->getInternalId()}"); + $dbForProject = new Database($pools->get($database)->pop()->getResource(), $cache); + $dbForProject->setNamespace("_{$project->getInternalId()}"); + $dbForProject->create(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); $audit = new Audit($dbForProject); diff --git a/app/controllers/api/teams.php b/app/controllers/api/teams.php index a42b9d317e..9f1aedf483 100644 --- a/app/controllers/api/teams.php +++ b/app/controllers/api/teams.php @@ -20,7 +20,6 @@ use Appwrite\Utopia\Response; use MaxMind\Db\Reader; use Utopia\App; use Utopia\Audit\Audit; -use Utopia\CLI\Console; use Utopia\Config\Config; use Utopia\Database\Database; use Utopia\Database\Document; @@ -36,9 +35,7 @@ use Utopia\Database\Validator\Key; use Utopia\Database\Validator\UID; use Utopia\Locale\Locale; use Utopia\Validator\Text; -use Utopia\Validator\Range; use Utopia\Validator\ArrayList; -use Utopia\Validator\WhiteList; App::post('/v1/teams') ->desc('Create Team') diff --git a/app/http.php b/app/http.php index 0b54f717c1..bfd7e7581c 100644 --- a/app/http.php +++ b/app/http.php @@ -2,7 +2,6 @@ require_once __DIR__ . '/../vendor/autoload.php'; -use Appwrite\Database\DatabasePool; use Appwrite\Utopia\Response; use Swoole\Process; use Swoole\Http\Server; @@ -23,6 +22,7 @@ use Utopia\Swoole\Files; use Appwrite\Utopia\Request; use Utopia\Logger\Log; use Utopia\Logger\Log\User; +use Utopia\Pools\Group; $http = new Server("0.0.0.0", App::getEnv('PORT', 80)); @@ -62,11 +62,8 @@ $http->on('start', function (Server $http) use ($payloadSize, $register) { go(function () use ($register, $app) { - $redis = $register->get('redisPool')->get(); - App::setResource('cache', fn() => $redis); - - $dbPool = $register->get('dbPool'); - App::setResource('dbPool', fn() => $dbPool); + $pools = $register->get('pools'); /** @var Group $pools */ + App::setResource('pools', fn() => $pools); // wait for database to be ready $attempts = 0; @@ -93,7 +90,8 @@ $http->on('start', function (Server $http) use ($payloadSize, $register) { $collections = Config::getParam('collections', []); try { - $redis->flushAll(); + $cache = $app->getResource('cache'); /** @var Utopia\Cache\Cache $cache */ + $cache->flush(); Console::success('[Setup] - Creating database: appwrite...'); $dbForConsole->create(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); } catch (\Exception $e) { @@ -217,7 +215,7 @@ $http->on('start', function (Server $http) use ($payloadSize, $register) { $dbForConsole->createCollection('bucket_' . $bucket->getInternalId(), $attributes, $indexes); } - $dbPool->reset(); + $pools->reclaim(); Console::success('[Setup] - Server database init completed...'); }); @@ -250,11 +248,8 @@ $http->on('request', function (SwooleRequest $swooleRequest, SwooleResponse $swo $app = new App('UTC'); - $redis = $register->get('redisPool')->get(); - App::setResource('cache', fn() => $redis); - - $dbPool = $register->get('dbPool'); - App::setResource('dbPool', fn() => $dbPool); + $pools = $register->get('pools'); + App::setResource('pools', fn() => $pools); try { Authorization::cleanRoles(); @@ -321,13 +316,6 @@ $http->on('request', function (SwooleRequest $swooleRequest, SwooleResponse $swo Console::error('[Error] File: ' . $th->getFile()); Console::error('[Error] Line: ' . $th->getLine()); - /** - * Reset Database connection if PDOException was thrown. - */ - if ($th instanceof PDOException) { - $db = null; - } - $swooleResponse->setStatusCode(500); $output = ((App::isDevelopment())) ? [ @@ -345,10 +333,7 @@ $http->on('request', function (SwooleRequest $swooleRequest, SwooleResponse $swo $swooleResponse->end(\json_encode($output)); } finally { - $dbPool->reset(); - /** @var RedisPool $redisPool */ - $redisPool = $register->get('redisPool'); - $redisPool->put($redis); + $pools->reclaim(); } }); diff --git a/app/init.php b/app/init.php index c64bb803e1..a08dcfd137 100644 --- a/app/init.php +++ b/app/init.php @@ -18,8 +18,6 @@ ini_set('display_startup_errors', 1); ini_set('default_socket_timeout', -1); error_reporting(E_ALL); -use Ahc\Jwt\JWT; -use Ahc\Jwt\JWTException; use Appwrite\Extend\Exception; use Appwrite\Auth\Auth; use Appwrite\SMS\Adapter\Mock; @@ -34,35 +32,28 @@ use Appwrite\Event\Database as EventDatabase; use Appwrite\Event\Event; use Appwrite\Event\Mail; use Appwrite\Event\Phone; +use Appwrite\Event\Delete; use Appwrite\Network\Validator\Email; use Appwrite\Network\Validator\IP; use Appwrite\Network\Validator\URL; use Appwrite\OpenSSL\OpenSSL; +use Appwrite\URL\URL as AppwriteURL; use Appwrite\Usage\Stats; use Appwrite\Utopia\View; use Utopia\App; +use Utopia\Validator\Range; +use Utopia\Validator\WhiteList; use Utopia\Database\ID; +use Utopia\Database\Document; +use Utopia\Database\Database; +use Utopia\Database\Query; +use Utopia\Database\Validator\Authorization; +use Utopia\Database\Validator\DatetimeValidator; +use Utopia\Database\Validator\Structure; use Utopia\Logger\Logger; use Utopia\Config\Config; use Utopia\Locale\Locale; use Utopia\Registry\Registry; -use MaxMind\Db\Reader; -use PHPMailer\PHPMailer\PHPMailer; -use Utopia\Database\Document; -use Utopia\Database\Database; -use Appwrite\Database\DatabasePool; -use Appwrite\Event\Delete; -use Utopia\Database\Validator\Structure; -use Utopia\Database\Validator\Authorization; -use Utopia\Cache\Cache; -use Utopia\Cache\Adapter\Redis as RedisCache; -use Utopia\Validator\Range; -use Utopia\Validator\WhiteList; -use Swoole\Database\RedisConfig; -use Swoole\Database\RedisPool; -use Utopia\Database\Adapter\MariaDB; -use Utopia\Database\Query; -use Utopia\Database\Validator\DatetimeValidator; use Utopia\Storage\Device; use Utopia\Storage\Storage; use Utopia\Storage\Device\Backblaze; @@ -71,6 +62,19 @@ use Utopia\Storage\Device\Local; use Utopia\Storage\Device\S3; use Utopia\Storage\Device\Linode; use Utopia\Storage\Device\Wasabi; +use Utopia\Cache\Adapter\Redis as RedisCache; +use Utopia\Cache\Adapter\Sharding; +use Utopia\Cache\Cache; +use Utopia\CLI\Console; +use Utopia\Database\Adapter\MariaDB; +use Utopia\Database\Adapter\MySQL; +use Utopia\Pools\Group; +use Utopia\Pools\Pool; +use Ahc\Jwt\JWT; +use Ahc\Jwt\JWTException; +use MaxMind\Db\Reader; +use PHPMailer\PHPMailer\PHPMailer; +use Swoole\Database\PDOProxy; const APP_NAME = 'Appwrite'; const APP_DOMAIN = 'appwrite.io'; @@ -493,53 +497,180 @@ $register->set('logger', function () { $adapter = new $classname($providerConfig); return new Logger($adapter); }); +$register->set('pools', function () { + $group = new Group(); -$register->set('dbPool', function () { - /** Parse the console databases */ - $consoleDB = App::getEnv('_APP_DB_CONSOLE', ''); - $consoleDB = explode(',', $consoleDB)[0]; - $consoleDB = explode('=', $consoleDB); - $name = $consoleDB[0]; - $dsn = $consoleDB[1]; - $consoleDBs[$name] = $dsn; + $fallbackForDB = AppwriteURL::unparse([ + 'scheme' => 'mariadb', + 'host' => App::getEnv('_APP_DB_HOST', 'mariadb'), + 'port' => App::getEnv('_APP_DB_PORT', '3306'), + 'user' => App::getEnv('_APP_DB_USER', ''), + 'pass' => App::getEnv('_APP_DB_PASS', ''), + ]); + $fallbackForRedis = AppwriteURL::unparse([ + 'scheme' => 'redis', + 'host' => App::getEnv('_APP_REDIS_HOST', 'redis'), + 'port' => App::getEnv('_APP_REDIS_PORT', '6379'), + 'user' => App::getEnv('_APP_REDIS_USER', ''), + 'pass' => App::getEnv('_APP_REDIS_PASS', ''), + ]); - /** Parse the project databases */ - $projectDBs = []; - $projectDB = App::getEnv('_APP_DB_PROJECT', ''); - $projectDB = explode(',', $projectDB); - foreach ($projectDB as $db) { - $db = explode('=', $db); - $name = $db[0]; - $dsn = $db[1]; - $projectDBs[$name] = $dsn; + $connections = [ + 'console' => [ + 'type' => 'database', + 'dsns' => App::getEnv('_APP_CONNECTIONS_DB_CONSOLE', $fallbackForDB), + 'multiple' => false, + 'schemes' => ['mariadb', 'mysql'], + ], + 'database' => [ + 'type' => 'database', + 'dsns' => App::getEnv('_APP_CONNECTIONS_DB_PROJECT', $fallbackForDB), + 'multiple' => true, + 'schemes' => ['mariadb', 'mysql'], + ], + 'queue' => [ + 'type' => 'queue', + 'dsns' => App::getEnv('_APP_CONNECTIONS_QUEUE', $fallbackForRedis), + 'multiple' => false, + 'schemes' => ['redis'], + ], + 'pubsub' => [ + 'type' => 'pubsub', + 'dsns' => App::getEnv('_APP_CONNECTIONS_PUBSUB', $fallbackForRedis), + 'multiple' => false, + 'schemes' => ['redis'], + ], + 'cache' => [ + 'type' => 'cache', + 'dsns' => App::getEnv('_APP_CONNECTIONS_CACHE', $fallbackForRedis), + 'multiple' => true, + 'schemes' => ['redis'], + ], + ]; + + foreach ($connections as $key => $connection) { + $type = $connection['type'] ?? ''; + $dsns = $connection['dsns'] ?? ''; + $multipe = $connection['multiple'] ?? false; + $schemes = $connection['schemes'] ?? []; + $config = []; + $dsns = explode(',', $connection['dsns'] ?? ''); + + foreach ($dsns as &$dsn) { + $dsn = explode('=', $dsn); + $name = ($multipe) ? $key . '_' . $dsn[0] : $key; + $dsn = $dsn[1] ?? ''; + $config[] = $name; + + if (empty($dsn)) { + //throw new Exception(Exception::GENERAL_SERVER_ERROR, "Missing value for DSN connection in {$key}"); + continue; + } + + $dsn = new DSN($dsn); + $dsnHost = $dsn->getHost(); + $dsnPort = $dsn->getPort(); + $dsnUser = $dsn->getUser(); + $dsnPass = $dsn->getPassword(); + $dsnScheme = $dsn->getScheme(); + $dsnDatabase = $dsn->getDatabase(); + + if (!in_array($dsnScheme, $schemes)) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, "Invalid console database scheme"); + } + + /** + * Get Resource + * + * Creation could be reused accross connection types like database, cache, queue, etc. + * + * Resource assignment to an adapter will happen below. + */ + + switch ($dsnScheme) { + case 'mysql': + case 'mariadb': + $resource = function () use ($dsnHost, $dsnPort, $dsnUser, $dsnPass, $dsnDatabase) { + return new PDOProxy(function () use ($dsnHost, $dsnPort, $dsnUser, $dsnPass, $dsnDatabase) { + return new PDO("mysql:host={$dsnHost};port={$dsnPort};dbname={$dsnDatabase};charset=utf8mb4", $dsnUser, $dsnPass, array( + PDO::ATTR_TIMEOUT => 3, // Seconds + PDO::ATTR_PERSISTENT => true, + PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC, + PDO::ATTR_ERRMODE => App::isDevelopment() ? PDO::ERRMODE_WARNING : PDO::ERRMODE_SILENT, // If in production mode, warnings are not displayed + PDO::ATTR_EMULATE_PREPARES => true, + PDO::ATTR_STRINGIFY_FETCHES => true + )); + }); + }; + break; + case 'redis': + $resource = function () use ($dsnHost, $dsnPort, $dsnPass) { + $redis = new Redis(); + @$redis->pconnect($dsnHost, (int)$dsnPort); + if ($dsnPass) { + $redis->auth($dsnPass); + } + $redis->setOption(Redis::OPT_READ_TIMEOUT, -1); + + return $redis; + }; + break; + + default: + throw new Exception(Exception::GENERAL_SERVER_ERROR, "Invalid scheme"); + break; + } + + $pool = new Pool($name, 64, function () use ($type, $resource, $dsn) { + // Get Adapter + $adapter = null; + + switch ($type) { + case 'database': + $adapter = match ($dsn->getScheme()) { + 'mariadb' => new MariaDB($resource()), + 'mysql' => new MySQL($resource()), + default => null + }; + + $adapter->setDefaultDatabase($dsn->getDatabase()); + + break; + case 'queue': + $adapter = $resource(); + break; + case 'pubsub': + $adapter = $resource(); + break; + case 'cache': + $adapter = match ($dsn->getScheme()) { + 'redis' => new RedisCache($resource()), + default => null + }; + break; + + default: + throw new Exception(Exception::GENERAL_SERVER_ERROR, "Server error: Missing adapter implementation."); + break; + } + + return $adapter; + }); + + $group->add($pool); + } + + Config::setParam('pools-' . $key, $config); } - $pool = new DatabasePool($consoleDBs, $projectDBs); - return $pool; -}); - -$register->set('redisPool', function () { - $redisHost = App::getEnv('_APP_REDIS_HOST', ''); - $redisPort = App::getEnv('_APP_REDIS_PORT', ''); - $redisUser = App::getEnv('_APP_REDIS_USER', ''); - $redisPass = App::getEnv('_APP_REDIS_PASS', ''); - $redisAuth = ''; - - if ($redisUser && $redisPass) { - $redisAuth = $redisUser . ':' . $redisPass; + try { + $group->fill(); + } catch (\Throwable $th) { + Console::error('Connection failure: ' . $th->getMessage()); } - $pool = new RedisPool( - (new RedisConfig()) - ->withHost($redisHost) - ->withPort($redisPort) - ->withAuth($redisAuth) - ->withDbIndex(0), - 64 - ); - - return $pool; + return $group; }); $register->set('influxdb', function () { // Register DB connection @@ -556,7 +687,7 @@ $register->set('influxdb', function () { return $client; }); $register->set('statsd', function () { - // Register DB connection + // Register DB connection $host = App::getEnv('_APP_STATSD_HOST', 'telegraf'); $port = App::getEnv('_APP_STATSD_PORT', 8125); @@ -596,14 +727,6 @@ $register->set('smtp', function () { $register->set('geodb', function () { return new Reader(__DIR__ . '/db/DBIP/dbip-country-lite-2022-06.mmdb'); }); -$register->set('cache', function () { - // This is usually for our workers or CLI commands scope - $redis = new Redis(); - $redis->pconnect(App::getEnv('_APP_REDIS_HOST', ''), App::getEnv('_APP_REDIS_PORT', '')); - $redis->setOption(Redis::OPT_READ_TIMEOUT, -1); - - return $redis; -}); /* * Localization @@ -901,22 +1024,51 @@ App::setResource('console', function () { ]); }, []); -App::setResource('dbForProject', function ($dbPool, $cache, Document $project) { - $database = $project->getAttribute('database', ''); - if (empty($database)) { - $database = $dbPool->getConsoleDB(); +App::setResource('dbForProject', function (Group $pools, Database $dbForConsole, Cache $cache, Document $project) { + if ($project->isEmpty() || $project->getId() === 'console') { + return $dbForConsole; } - $pdo = $dbPool->getPDOFromPool($database); - $database = DatabasePool::getDatabase($pdo->getConnection(), $cache, "_{$project->getInternalId()}"); - return $database; -}, ['dbPool', 'cache', 'project']); -App::setResource('dbForConsole', function ($dbPool, $cache) { - $database = $dbPool->getConsoleDB(); - $pdo = $dbPool->getPDOFromPool($database); - $database = DatabasePool::getDatabase($pdo->getConnection(), $cache, '_console'); + $dbAdapter = $pools + ->get($project->getAttribute('database')) + ->pop() + ->getResource() + ; + + $database = new Database($dbAdapter, $cache); + $database->setNamespace('_' . $project->getInternalId()); + return $database; -}, ['dbPool', 'cache']); +}, ['pools', 'dbForConsole', 'cache', 'project']); + +App::setResource('dbForConsole', function (Group $pools, Cache $cache) { + $dbAdapter = $pools + ->get('console') + ->pop() + ->getResource() + ; + + $database = new Database($dbAdapter, $cache); + + $database->setNamespace('console'); + + return $database; +}, ['pools', 'cache']); + +App::setResource('cache', function (Group $pools) { + $list = Config::getParam('pools-cache', []); + $adapters = []; + + foreach ($list as $value) { + $adapters[] = $pools + ->get($value) + ->pop() + ->getResource() + ; + } + + return new Cache(new Sharding($adapters)); +}, ['pools']); App::setResource('deviceLocal', function () { return new Local(); diff --git a/app/preload.php b/app/preload.php index bf8b0bfd1d..4935db3da4 100644 --- a/app/preload.php +++ b/app/preload.php @@ -35,6 +35,7 @@ foreach ( realpath(__DIR__ . '/../vendor/symfony'), realpath(__DIR__ . '/../vendor/mongodb'), realpath(__DIR__ . '/../vendor/utopia-php/websocket'), // TODO: remove workerman autoload + realpath(__DIR__ . '/../vendor/utopia-php/cache'), // TODO: remove memcached autoload ] as $key => $value ) { if ($value !== false) { diff --git a/app/realtime.php b/app/realtime.php index ab39a18a57..8cd28c193f 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -1,7 +1,6 @@ get('pools'); /** @var \Utopia\Pools\Group $pools */ + + $dbAdapter = $pools + ->get('console') + ->pop() + ->getResource() + ; + + $database = new Database($dbAdapter, getCache()); + + $database->setNamespace('console'); + + return $database; +} + +function getProjectDB(Document $project): Database +{ + global $register; + + $pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */ + + if ($project->isEmpty() || $project->getId() === 'console') { + return getConsoleDB(); + } + + $dbAdapter = $pools + ->get($project->getAttribute('database')) + ->pop() + ->getResource() + ; + + $database = new Database($dbAdapter, getCache()); + $database->setNamespace('_' . $project->getInternalId()); + + return $database; +} + +function getCache(): Cache +{ + global $register; + + $pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */ + + $list = Config::getParam('pools-cache', []); + $adapters = []; + + foreach ($list as $value) { + $adapters[] = $pools + ->get($value) + ->pop() + ->getResource() + ; + } + + return new Cache(new Sharding($adapters)); +} + $realtime = new Realtime(); /** @@ -92,38 +156,6 @@ $logError = function (Throwable $error, string $action) use ($register) { $server->error($logError); -function getDatabase(Registry &$register, string $projectId) -{ - $redis = $register->get('redisPool')->get(); - $dbPool = $register->get('dbPool'); - - /** Get the console DB */ - $database = $dbPool->getConsoleDB(); - $pdo = $dbPool->getPDOFromPool($database); - $database = DatabasePool::wait( - DatabasePool::getDatabase($pdo->getConnection(), $redis, '_console'), - 'realtime' - ); - - if ($projectId !== 'console') { - $project = Authorization::skip(fn() => $database->getDocument('projects', $projectId)); - $database = $project->getAttribute('database', ''); - $pdo = $dbPool->getPDOFromPool($database); - $database = DatabasePool::wait( - DatabasePool::getDatabase($pdo->getConnection(), $redis, "_{$project->getInternalId()}"), - 'realtime' - ); - } - - return [ - $database, - function () use ($register, $redis) { - $register->get('dbPool')->reset(); - $register->get('redisPool')->put($redis); - } - ]; -} - $server->onStart(function () use ($stats, $register, $containerId, &$statsDocument, $logError) { sleep(5); // wait for the initial database schema to be ready Console::success('Server started successfully'); @@ -133,7 +165,8 @@ $server->onStart(function () use ($stats, $register, $containerId, &$statsDocume */ go(function () use ($register, $containerId, &$statsDocument) { $attempts = 0; - [$database, $returnDatabase] = getDatabase($register, 'console'); + $database = getConsoleDB(); + do { try { $attempts++; @@ -153,7 +186,7 @@ $server->onStart(function () use ($stats, $register, $containerId, &$statsDocume sleep(DATABASE_RECONNECT_SLEEP); } } while (true); - call_user_func($returnDatabase); + $register->get('pools')->reclaim(); }); /** @@ -169,7 +202,7 @@ $server->onStart(function () use ($stats, $register, $containerId, &$statsDocume } try { - [$database, $returnDatabase] = getDatabase($register, 'console'); + $database = getConsoleDB(); $statsDocument ->setAttribute('timestamp', DateTime::now()) @@ -179,7 +212,7 @@ $server->onStart(function () use ($stats, $register, $containerId, &$statsDocume } catch (\Throwable $th) { call_user_func($logError, $th, "updateWorkerDocument"); } finally { - call_user_func($returnDatabase); + $register->get('pools')->reclaim(); } }); }); @@ -195,7 +228,7 @@ $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats, * Sending current connections to project channels on the console project every 5 seconds. */ if ($realtime->hasSubscriber('console', Role::users()->toString(), 'project')) { - [$database, $returnDatabase] = getDatabase($register, '_console'); + $database = getConsoleDB(); $payload = []; @@ -240,7 +273,7 @@ $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats, ])); } - call_user_func($returnDatabase); + $register->get('pools')->reclaim(); } /** * Sending test message for SDK E2E tests every 5 seconds. @@ -275,8 +308,7 @@ $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats, } $start = time(); - /** @var Redis $redis */ - $redis = $register->get('redisPool')->get(); + $redis = $register->get('pools')->get('pubsub')->pop()->getResource(); /** @var Redis $redis */ $redis->setOption(Redis::OPT_READ_TIMEOUT, -1); if ($redis->ping(true)) { @@ -295,9 +327,9 @@ $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats, if ($realtime->hasSubscriber($projectId, 'user:' . $userId)) { $connection = array_key_first(reset($realtime->subscriptions[$projectId]['user:' . $userId])); - [$consoleDatabase, $returnConsoleDatabase] = getDatabase($register, 'console'); + $consoleDatabase = getConsoleDB(); $project = Authorization::skip(fn() => $consoleDatabase->getDocument('projects', $projectId)); - [$database, $returnDatabase] = getDatabase($register, $project->getId()); + $database = getProjectDB($project); $user = $database->getDocument('users', $userId); @@ -305,8 +337,7 @@ $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats, $realtime->subscribe($projectId, $connection, $roles, $realtime->connections[$connection]['channels']); - call_user_func($returnDatabase); - call_user_func($returnConsoleDatabase); + $register->get('pools')->reclaim(); } } @@ -334,7 +365,7 @@ $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats, call_user_func($logError, $th, "pubSubConnection"); Console::error('Pub/sub error: ' . $th->getMessage()); - $register->get('redisPool')->put($redis); + $register->get('pools')->reclaim(); $attempts++; sleep(DATABASE_RECONNECT_SLEEP); continue; @@ -349,15 +380,9 @@ $server->onOpen(function (int $connection, SwooleRequest $request) use ($server, $request = new Request($request); $response = new Response(new SwooleResponse()); - /** @var PDO $db */ - $dbPool = $register->get('dbPool'); - /** @var Redis $redis */ - $redis = $register->get('redisPool')->get(); - Console::info("Connection open (user: {$connection})"); - App::setResource('dbPool', fn() => $dbPool); - App::setResource('cache', fn() => $redis); + App::setResource('pools', fn() => $register->get('pools')); App::setResource('request', fn() => $request); App::setResource('response', fn() => $response); @@ -372,13 +397,9 @@ $server->onOpen(function (int $connection, SwooleRequest $request) use ($server, throw new Exception('Missing or unknown project ID', 1008); } - $dbForProject = $app->getResource('dbForProject'); - - /** @var \Utopia\Database\Document $console */ - $console = $app->getResource('console'); - - /** @var \Utopia\Database\Document $user */ - $user = $app->getResource('user'); + $dbForProject = getProjectDB($project); + $console = $app->getResource('console'); /** @var \Utopia\Database\Document $console */ + $user = $app->getResource('user'); /** @var \Utopia\Database\Document $user */ /* * Abuse Check @@ -456,36 +477,20 @@ $server->onOpen(function (int $connection, SwooleRequest $request) use ($server, Console::error('[Error] Code: ' . $response['data']['code']); Console::error('[Error] Message: ' . $response['data']['message']); } - - if ($th instanceof PDOException) { - $db = null; - } } finally { - /** - * Put used PDO and Redis Connections back into their pools. - */ - $dbPool->reset(); - $register->get('redisPool')->put($redis); + $register->get('pools')->reclaim(); } }); $server->onMessage(function (int $connection, string $message) use ($server, $register, $realtime, $containerId) { try { $response = new Response(new SwooleResponse()); - $redis = $register->get('redisPool')->get(); - $dbPool = $register->get('dbPool'); $projectId = $realtime->connections[$connection]['projectId']; - - /** Get the console DB */ - $database = $dbPool->getConsoleDB(); - $pdo = $dbPool->getPDOFromPool($database); - $database = DatabasePool::getDatabase($pdo->getConnection(), $redis, '_console'); + $database = getConsoleDB(); if ($projectId !== 'console') { $project = Authorization::skip(fn() => $database->getDocument('projects', $projectId)); - $database = $project->getAttribute('database', ''); - $pdo = $dbPool->getPDOFromPool($database); - $database = DatabasePool::getDatabase($pdo->getConnection(), $redis, "_{$project->getInternalId()}"); + $database = getProjectDB($project); } /* @@ -569,8 +574,7 @@ $server->onMessage(function (int $connection, string $message) use ($server, $re $server->close($connection, $th->getCode()); } } finally { - $dbPool->reset(); - $register->get('redisPool')->put($redis); + $register->get('pools')->reclaim(); } }); diff --git a/app/tasks/doctor.php b/app/tasks/doctor.php index 634381ba41..daeaaa3f52 100644 --- a/app/tasks/doctor.php +++ b/app/tasks/doctor.php @@ -8,6 +8,7 @@ use Utopia\Storage\Device\Local; use Utopia\Storage\Storage; use Utopia\App; use Utopia\CLI\Console; +use Utopia\Config\Config; use Utopia\Domains\Domain; $cli @@ -21,7 +22,7 @@ $cli Console::log("\n" . '👩‍⚕️ Running ' . APP_NAME . ' Doctor for version ' . App::getEnv('_APP_VERSION', 'UNKNOWN') . ' ...' . "\n"); - Console::log('Checking for production best practices...'); + Console::log('[Settings]'); $domain = new Domain(App::getEnv('_APP_DOMAIN')); @@ -77,7 +78,6 @@ $cli Console::log('🟢 HTTPS force option is enabled'); } - $providerName = App::getEnv('_APP_LOGGING_PROVIDER', ''); $providerConfig = App::getEnv('_APP_LOGGING_CONFIG', ''); @@ -90,32 +90,55 @@ $cli \sleep(0.2); try { - Console::log("\n" . 'Checking connectivity...'); + Console::log("\n" . '[Connectivity]'); } catch (\Throwable $th) { //throw $th; } - try { - $dbPool = $register->get('dbPool'); /* @var $dbPool DatabasePool */ - $database = $dbPool->getConsoleDB(); - $pdo = $dbPool->getPDO($database); - Console::success('Database............connected 👍'); - } catch (\Throwable $th) { - Console::error('Database.........disconnected 👎'); + $pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */ + + $configs = [ + 'Console.DB' => Config::getParam('pools-console'), + 'Projects.DB' => Config::getParam('pools-database'), + ]; + + foreach ($configs as $key => $config) { + foreach ($config as $database) { + try { + $adapter = $pools->get($database)->pop()->getResource(); + + if ($adapter->ping()) { + Console::success('🟢 ' . str_pad("{$key}({$database})", 50, '.') . 'connected'); + } else { + Console::error('🔴 ' . str_pad("{$key}({$database})", 47, '.') . 'disconnected'); + } + } catch (\Throwable $th) { + Console::error('🔴 ' . str_pad("{$key}.({$database})", 47, '.') . 'disconnected'); + } + } } - try { - $register->get('cache'); - Console::success('Queue...............connected 👍'); - } catch (\Throwable $th) { - Console::error('Queue............disconnected 👎'); - } + $pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */ + $configs = [ + 'Cache' => Config::getParam('pools-cache'), + 'Queue' => Config::getParam('pools-queue'), + 'PubSub' => Config::getParam('pools-pubsub'), + ]; - try { - $register->get('cache'); - Console::success('Cache...............connected 👍'); - } catch (\Throwable $th) { - Console::error('Cache............disconnected 👎'); + foreach ($configs as $key => $config) { + foreach ($config as $pool) { + try { + $adapter = $pools->get($pool)->pop()->getResource(); + + if ($adapter->ping()) { + Console::success('🟢 ' . str_pad("{$key}({$pool})", 50, '.') . 'connected'); + } else { + Console::error('🔴 ' . str_pad("{$key}({$pool})", 47, '.') . 'disconnected'); + } + } catch (\Throwable $th) { + Console::error('🔴 ' . str_pad("{$key}({$pool})", 47, '.') . 'disconnected'); + } + } } if (App::getEnv('_APP_STORAGE_ANTIVIRUS') === 'enabled') { // Check if scans are enabled @@ -126,12 +149,12 @@ $cli ); if ((@$antivirus->ping())) { - Console::success('Antivirus...........connected 👍'); + Console::success('🟢 ' . str_pad("Antivirus", 50, '.') . 'connected'); } else { - Console::error('Antivirus........disconnected 👎'); + Console::error('🔴 ' . str_pad("Antivirus", 47, '.') . 'disconnected'); } } catch (\Throwable $th) { - Console::error('Antivirus........disconnected 👎'); + Console::error('🔴 ' . str_pad("Antivirus", 47, '.') . 'disconnected'); } } @@ -144,35 +167,35 @@ $cli $mail->AltBody = 'Hello World'; $mail->send(); - Console::success('SMTP................connected 👍'); + Console::success('🟢 ' . str_pad("SMTP", 50, '.') . 'connected'); } catch (\Throwable $th) { - Console::error('SMTP.............disconnected 👎'); + Console::error('🔴 ' . str_pad("SMTP", 47, '.') . 'disconnected'); } $host = App::getEnv('_APP_STATSD_HOST', 'telegraf'); $port = App::getEnv('_APP_STATSD_PORT', 8125); if ($fp = @\fsockopen('udp://' . $host, $port, $errCode, $errStr, 2)) { - Console::success('StatsD..............connected 👍'); + Console::success('🟢 ' . str_pad("StatsD", 50, '.') . 'connected'); \fclose($fp); } else { - Console::error('StatsD...........disconnected 👎'); + Console::error('🔴 ' . str_pad("StatsD", 47, '.') . 'disconnected'); } $host = App::getEnv('_APP_INFLUXDB_HOST', ''); $port = App::getEnv('_APP_INFLUXDB_PORT', ''); if ($fp = @\fsockopen($host, $port, $errCode, $errStr, 2)) { - Console::success('InfluxDB............connected 👍'); + Console::success('🟢 ' . str_pad("InfluxDB", 50, '.') . 'connected'); \fclose($fp); } else { - Console::error('InfluxDB.........disconnected 👎'); + Console::error('🔴 ' . str_pad("InfluxDB", 47, '.') . 'disconnected'); } \sleep(0.2); Console::log(''); - Console::log('Checking volumes...'); + Console::log('[Volumes]'); foreach ( [ @@ -200,7 +223,7 @@ $cli \sleep(0.2); Console::log(''); - Console::log('Checking disk space usage...'); + Console::log('[Disk Space]'); foreach ( [ diff --git a/app/tasks/maintenance.php b/app/tasks/maintenance.php index af4719d119..1c16ca6911 100644 --- a/app/tasks/maintenance.php +++ b/app/tasks/maintenance.php @@ -3,7 +3,6 @@ global $cli; use Appwrite\Auth\Auth; -use Appwrite\Database\DatabasePool; use Appwrite\Event\Certificate; use Appwrite\Event\Delete; use Utopia\App; @@ -16,8 +15,6 @@ $cli ->task('maintenance') ->desc('Schedules maintenance tasks and publishes them to resque') ->action(function () { - global $register; - Console::title('Maintenance V1'); Console::success(APP_NAME . ' maintenance process v1 has started'); @@ -115,16 +112,8 @@ $cli $usageStatsRetention1d = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_USAGE_1D', '8640000'); // 100 days $cacheRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_CACHE', '2592000'); // 30 days - Console::loop(function () use ($register, $interval, $executionLogsRetention, $abuseLogsRetention, $auditLogRetention, $usageStatsRetention30m, $usageStatsRetention1d, $cacheRetention) { - $redis = $register->get('cache'); - $dbPool = $register->get('dbPool'); - - $database = $dbPool->getConsoleDB(); - $pdo = $dbPool->getPDO($database); - $database = DatabasePool::wait( - DatabasePool::getDatabase($pdo, $redis, '_console'), - 'certificates', - ); + Console::loop(function () use ($interval, $executionLogsRetention, $abuseLogsRetention, $auditLogRetention, $usageStatsRetention30m, $usageStatsRetention1d, $cacheRetention) { + $database = getConsoleDB(); $time = DateTime::now(); diff --git a/app/tasks/specs.php b/app/tasks/specs.php index 755662bbc7..8d1bc5039f 100644 --- a/app/tasks/specs.php +++ b/app/tasks/specs.php @@ -9,8 +9,12 @@ use Appwrite\Specification\Specification; use Appwrite\Utopia\Response; use Swoole\Http\Response as HttpResponse; use Utopia\App; +use Utopia\Cache\Adapter\None; +use Utopia\Cache\Cache; use Utopia\CLI\Console; use Utopia\Config\Config; +use Utopia\Database\Adapter\MySQL; +use Utopia\Database\Database; use Utopia\Request; use Utopia\Validator\WhiteList; @@ -19,16 +23,15 @@ $cli ->param('version', 'latest', new Text(16), 'Spec version', true) ->param('mode', 'normal', new WhiteList(['normal', 'mocks']), 'Spec Mode', true) ->action(function ($version, $mode) use ($register) { - $consoleDB = $register->get('dbPool')->getConsoleDB(); - $redis = $register->get('cache'); $appRoutes = App::getRoutes(); $response = new Response(new HttpResponse()); $mocks = ($mode === 'mocks'); + // Mock dependencies App::setResource('request', fn () => new Request()); App::setResource('response', fn () => $response); - App::setResource('consoleDB', fn () => $consoleDB); - App::setResource('cache', fn () => $redis); + App::setResource('dbForConsole', fn () => new Database(new MySQL(''), new Cache(new None()))); + App::setResource('dbForProject', fn () => new Database(new MySQL(''), new Cache(new None()))); $platforms = [ 'client' => APP_PLATFORM_CLIENT, diff --git a/app/tasks/usage.php b/app/tasks/usage.php index a0b7a75015..d1aeab2e84 100644 --- a/app/tasks/usage.php +++ b/app/tasks/usage.php @@ -2,7 +2,6 @@ global $cli, $register; -use Appwrite\Database\DatabasePool; use Appwrite\Usage\Calculators\Aggregator; use Appwrite\Usage\Calculators\Database; use Appwrite\Usage\Calculators\TimeSeries; @@ -11,39 +10,12 @@ use Utopia\App; use Utopia\CLI\Console; use Utopia\Database\Database as UtopiaDatabase; use Utopia\Database\Validator\Authorization; -use Utopia\Registry\Registry; use Utopia\Logger\Log; use Utopia\Validator\WhiteList; Authorization::disable(); Authorization::setDefaultStatus(false); -function getInfluxDB(Registry &$register): InfluxDatabase -{ - /** @var InfluxDB\Client $client */ - $client = $register->get('influxdb'); - $attempts = 0; - $max = 10; - $sleep = 1; - - do { // check if telegraf database is ready - try { - $attempts++; - $database = $client->selectDB('telegraf'); - if (in_array('telegraf', $client->listDatabases())) { - break; // leave the do-while if successful - } - } catch (\Throwable $th) { - Console::warning("InfluxDB not ready. Retrying connection ({$attempts})..."); - if ($attempts >= $max) { - throw new \Exception('InfluxDB database not ready yet'); - } - sleep($sleep); - } - } while ($attempts < $max); - return $database; -} - $logError = function (Throwable $error, string $action = 'syncUsageStats') use ($register) { $logger = $register->get('logger'); @@ -78,7 +50,6 @@ $logError = function (Throwable $error, string $action = 'syncUsageStats') use ( Console::warning($error->getTraceAsString()); }; - function aggregateTimeseries(UtopiaDatabase $database, InfluxDatabase $influxDB, callable $logError): void { $interval = (int) App::getEnv('_APP_USAGE_TIMESERIES_INTERVAL', '30'); // 30 seconds (by default) @@ -120,21 +91,12 @@ $cli ->task('usage') ->param('type', 'timeseries', new WhiteList(['timeseries', 'database'])) ->desc('Schedules syncing data from influxdb to Appwrite console db') - ->action(function (string $type) use ($register, $logError) { + ->action(function (string $type) use ($logError) { Console::title('Usage Aggregation V1'); Console::success(APP_NAME . ' usage aggregation process v1 has started'); - $redis = $register->get('cache'); - $dbPool = $register->get('dbPool'); - - $database = $dbPool->getConsoleDB(); - $pdo = $dbPool->getPDO($database); - $database = DatabasePool::wait( - DatabasePool::getDatabase($pdo, $redis, '_console'), - 'projects', - ); - - $influxDB = getInfluxDB($register); + $database = getConsoleDB(); + $influxDB = getInfluxDB(); switch ($type) { case 'timeseries': diff --git a/app/views/install/compose.phtml b/app/views/install/compose.phtml index 944d3eab0a..1d7fdcd046 100644 --- a/app/views/install/compose.phtml +++ b/app/views/install/compose.phtml @@ -88,12 +88,15 @@ services: - _APP_OPENSSL_KEY_V1 - _APP_DOMAIN - _APP_DOMAIN_TARGET + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS - _APP_REDIS_HOST - _APP_REDIS_PORT - _APP_REDIS_USER - _APP_REDIS_PASS - - _APP_DB_CONSOLE - - _APP_DB_PROJECT - _APP_SMTP_HOST - _APP_SMTP_PORT - _APP_SMTP_SECURE @@ -181,10 +184,15 @@ services: - _APP_WORKER_PER_CORE - _APP_OPTIONS_ABUSE - _APP_OPENSSL_KEY_V1 + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS - _APP_REDIS_HOST - _APP_REDIS_PORT - - _APP_DB_CONSOLE - - _APP_DB_PROJECT + - _APP_REDIS_USER + - _APP_REDIS_PASS - _APP_USAGE_STATS - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG @@ -203,12 +211,15 @@ services: environment: - _APP_ENV - _APP_OPENSSL_KEY_V1 + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS - _APP_REDIS_HOST - _APP_REDIS_PORT - _APP_REDIS_USER - _APP_REDIS_PASS - - _APP_DB_CONSOLE - - _APP_DB_PROJECT - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG @@ -254,12 +265,15 @@ services: environment: - _APP_ENV - _APP_OPENSSL_KEY_V1 + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS - _APP_REDIS_HOST - _APP_REDIS_PORT - _APP_REDIS_USER - _APP_REDIS_PASS - - _APP_DB_CONSOLE - - _APP_DB_PROJECT - _APP_STORAGE_DEVICE - _APP_STORAGE_S3_ACCESS_KEY - _APP_STORAGE_S3_SECRET @@ -300,12 +314,15 @@ services: environment: - _APP_ENV - _APP_OPENSSL_KEY_V1 + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS - _APP_REDIS_HOST - _APP_REDIS_PORT - _APP_REDIS_USER - _APP_REDIS_PASS - - _APP_DB_CONSOLE - - _APP_DB_PROJECT - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG @@ -325,12 +342,15 @@ services: - _APP_OPENSSL_KEY_V1 - _APP_EXECUTOR_SECRET - _APP_EXECUTOR_HOST + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS - _APP_REDIS_HOST - _APP_REDIS_PORT - _APP_REDIS_USER - _APP_REDIS_PASS - - _APP_DB_CONSOLE - - _APP_DB_PROJECT - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG @@ -354,12 +374,15 @@ services: - _APP_DOMAIN - _APP_DOMAIN_TARGET - _APP_SYSTEM_SECURITY_EMAIL_ADDRESS + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS - _APP_REDIS_HOST - _APP_REDIS_PORT - _APP_REDIS_USER - _APP_REDIS_PASS - - _APP_DB_CONSOLE - - _APP_DB_PROJECT - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG @@ -378,12 +401,15 @@ services: environment: - _APP_ENV - _APP_OPENSSL_KEY_V1 + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS - _APP_REDIS_HOST - _APP_REDIS_PORT - _APP_REDIS_USER - _APP_REDIS_PASS - - _APP_DB_CONSOLE - - _APP_DB_PROJECT - _APP_FUNCTIONS_TIMEOUT - _APP_EXECUTOR_SECRET - _APP_EXECUTOR_HOST @@ -511,12 +537,15 @@ services: - _APP_OPENSSL_KEY_V1 - _APP_DOMAIN - _APP_DOMAIN_TARGET + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS - _APP_REDIS_HOST - _APP_REDIS_PORT - _APP_REDIS_USER - _APP_REDIS_PASS - - _APP_DB_CONSOLE - - _APP_DB_PROJECT - _APP_MAINTENANCE_INTERVAL - _APP_MAINTENANCE_RETENTION_EXECUTION - _APP_MAINTENANCE_RETENTION_CACHE @@ -539,16 +568,19 @@ services: environment: - _APP_ENV - _APP_OPENSSL_KEY_V1 - - _APP_DB_CONSOLE - - _APP_DB_PROJECT - - _APP_INFLUXDB_HOST - - _APP_INFLUXDB_PORT - - _APP_USAGE_TIMESERIES_INTERVAL - - _APP_USAGE_DATABASE_INTERVAL + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS - _APP_REDIS_HOST - _APP_REDIS_PORT - _APP_REDIS_USER - _APP_REDIS_PASS + - _APP_INFLUXDB_HOST + - _APP_INFLUXDB_PORT + - _APP_USAGE_TIMESERIES_INTERVAL + - _APP_USAGE_DATABASE_INTERVAL - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG @@ -573,14 +605,14 @@ services: - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS - - _APP_INFLUXDB_HOST - - _APP_INFLUXDB_PORT - - _APP_USAGE_TIMESERIES_INTERVAL - - _APP_USAGE_DATABASE_INTERVAL - _APP_REDIS_HOST - _APP_REDIS_PORT - _APP_REDIS_USER - _APP_REDIS_PASS + - _APP_INFLUXDB_HOST + - _APP_INFLUXDB_PORT + - _APP_USAGE_TIMESERIES_INTERVAL + - _APP_USAGE_DATABASE_INTERVAL - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG diff --git a/app/workers/audits.php b/app/workers/audits.php index 24812c7dbf..90ac020536 100644 --- a/app/workers/audits.php +++ b/app/workers/audits.php @@ -1,6 +1,5 @@ =8.0" }, "require-dev": { + "laravel/pint": "1.2.*", "phpunit/phpunit": "^9.3", "vimeo/psalm": "4.13.1" }, @@ -1928,12 +1930,6 @@ "license": [ "MIT" ], - "authors": [ - { - "name": "Eldad Fux", - "email": "eldad@appwrite.io" - } - ], "description": "A simple cache library to manage application cache storing, loading and purging", "keywords": [ "cache", @@ -1944,9 +1940,9 @@ ], "support": { "issues": "https://github.com/utopia-php/cache/issues", - "source": "https://github.com/utopia-php/cache/tree/0.6.1" + "source": "https://github.com/utopia-php/cache/tree/0.8.0" }, - "time": "2022-08-10T08:12:46+00:00" + "time": "2022-10-16T16:48:09+00:00" }, { "name": "utopia-php/cli", @@ -2054,16 +2050,16 @@ }, { "name": "utopia-php/database", - "version": "0.26.0", + "version": "dev-feat-update-cache-lib", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "d172af2541137c83a86d066f82f48914b5a3a610" + "reference": "44ae47dfd49c9c7c0cba29f6867347e25c23b57b" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/d172af2541137c83a86d066f82f48914b5a3a610", - "reference": "d172af2541137c83a86d066f82f48914b5a3a610", + "url": "https://api.github.com/repos/utopia-php/database/zipball/44ae47dfd49c9c7c0cba29f6867347e25c23b57b", + "reference": "44ae47dfd49c9c7c0cba29f6867347e25c23b57b", "shasum": "" }, "require": { @@ -2072,7 +2068,7 @@ "ext-redis": "*", "mongodb/mongodb": "1.8.0", "php": ">=8.0", - "utopia-php/cache": "0.6.*", + "utopia-php/cache": "0.8.*", "utopia-php/framework": "0.*.*" }, "require-dev": { @@ -2092,16 +2088,6 @@ "license": [ "MIT" ], - "authors": [ - { - "name": "Eldad Fux", - "email": "eldad@appwrite.io" - }, - { - "name": "Brandon Leckemby", - "email": "brandon@appwrite.io" - } - ], "description": "A simple library to manage application persistency using multiple database adapters", "keywords": [ "database", @@ -2112,9 +2098,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/0.26.0" + "source": "https://github.com/utopia-php/database/tree/feat-update-cache-lib" }, - "time": "2022-10-03T17:12:01+00:00" + "time": "2022-10-16T17:35:26+00:00" }, { "name": "utopia-php/domains", @@ -2443,6 +2429,59 @@ }, "time": "2022-07-13T16:47:18+00:00" }, + { + "name": "utopia-php/pools", + "version": "0.1.0", + "source": { + "type": "git", + "url": "https://github.com/utopia-php/pools.git", + "reference": "5a467a569a80aefc846a97dc195b4adc2fd71805" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/utopia-php/pools/zipball/5a467a569a80aefc846a97dc195b4adc2fd71805", + "reference": "5a467a569a80aefc846a97dc195b4adc2fd71805", + "shasum": "" + }, + "require": { + "ext-mongodb": "*", + "ext-pdo": "*", + "ext-redis": "*", + "php": ">=8.0" + }, + "require-dev": { + "phpunit/phpunit": "^9.4", + "vimeo/psalm": "4.0.1" + }, + "type": "library", + "autoload": { + "psr-4": { + "Utopia\\Pools\\": "src/Pools" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Team Appwrite", + "email": "team@appwrite.io" + } + ], + "description": "A simple library to manage connection pools", + "keywords": [ + "framework", + "php", + "pools", + "utopia" + ], + "support": { + "issues": "https://github.com/utopia-php/pools/issues", + "source": "https://github.com/utopia-php/pools/tree/0.1.0" + }, + "time": "2022-10-11T19:31:07+00:00" + }, { "name": "utopia-php/preloader", "version": "0.2.4", @@ -3535,16 +3574,16 @@ }, { "name": "phpunit/php-code-coverage", - "version": "9.2.16", + "version": "9.2.17", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/php-code-coverage.git", - "reference": "2593003befdcc10db5e213f9f28814f5aa8ac073" + "reference": "aa94dc41e8661fe90c7316849907cba3007b10d8" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/2593003befdcc10db5e213f9f28814f5aa8ac073", - "reference": "2593003befdcc10db5e213f9f28814f5aa8ac073", + "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/aa94dc41e8661fe90c7316849907cba3007b10d8", + "reference": "aa94dc41e8661fe90c7316849907cba3007b10d8", "shasum": "" }, "require": { @@ -3600,7 +3639,7 @@ ], "support": { "issues": "https://github.com/sebastianbergmann/php-code-coverage/issues", - "source": "https://github.com/sebastianbergmann/php-code-coverage/tree/9.2.16" + "source": "https://github.com/sebastianbergmann/php-code-coverage/tree/9.2.17" }, "funding": [ { @@ -3608,7 +3647,7 @@ "type": "github" } ], - "time": "2022-08-20T05:26:47+00:00" + "time": "2022-08-30T12:24:04+00:00" }, { "name": "phpunit/php-file-iterator", @@ -5357,9 +5396,18 @@ "time": "2022-09-28T08:42:51+00:00" } ], - "aliases": [], + "aliases": [ + { + "package": "utopia-php/database", + "version": "dev-feat-update-cache-lib", + "alias": "0.26.1", + "alias_normalized": "0.26.1.0" + } + ], "minimum-stability": "stable", - "stability-flags": [], + "stability-flags": { + "utopia-php/database": 20 + }, "prefer-stable": false, "prefer-lowest": false, "platform": { diff --git a/docker-compose.yml b/docker-compose.yml index 963c677119..ea70cf331a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -133,12 +133,20 @@ services: - _APP_OPENSSL_KEY_V1 - _APP_DOMAIN - _APP_DOMAIN_TARGET + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS - _APP_REDIS_HOST - _APP_REDIS_PORT - _APP_REDIS_USER - _APP_REDIS_PASS - - _APP_DB_PROJECT - - _APP_DB_CONSOLE + - _APP_CONNECTIONS_DB_PROJECT + - _APP_CONNECTIONS_DB_CONSOLE + - _APP_CONNECTIONS_CACHE + - _APP_CONNECTIONS_QUEUE + - _APP_CONNECTIONS_PUBSUB - _APP_SMTP_HOST - _APP_SMTP_PORT - _APP_SMTP_SECURE @@ -211,10 +219,19 @@ services: - _APP_WORKER_PER_CORE - _APP_OPTIONS_ABUSE - _APP_OPENSSL_KEY_V1 + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS - _APP_REDIS_HOST - _APP_REDIS_PORT - - _APP_DB_CONSOLE - - _APP_DB_PROJECT + - _APP_REDIS_USER + - _APP_REDIS_PASS + - _APP_CONNECTIONS_DB_CONSOLE + - _APP_CONNECTIONS_DB_PROJECT + - _APP_CONNECTIONS_CACHE + - _APP_CONNECTIONS_PUBSUB - _APP_USAGE_STATS - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG @@ -235,12 +252,19 @@ services: environment: - _APP_ENV - _APP_OPENSSL_KEY_V1 + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS - _APP_REDIS_HOST - _APP_REDIS_PORT - _APP_REDIS_USER - _APP_REDIS_PASS - - _APP_DB_CONSOLE - - _APP_DB_PROJECT + - _APP_CONNECTIONS_DB_CONSOLE + - _APP_CONNECTIONS_DB_PROJECT + - _APP_CONNECTIONS_CACHE + - _APP_CONNECTIONS_QUEUE - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG @@ -266,6 +290,7 @@ services: - _APP_REDIS_PORT - _APP_REDIS_USER - _APP_REDIS_PASS + - _APP_CONNECTIONS_QUEUE - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG @@ -290,12 +315,19 @@ services: environment: - _APP_ENV - _APP_OPENSSL_KEY_V1 + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS - _APP_REDIS_HOST - _APP_REDIS_PORT - _APP_REDIS_USER - _APP_REDIS_PASS - - _APP_DB_CONSOLE - - _APP_DB_PROJECT + - _APP_CONNECTIONS_DB_CONSOLE + - _APP_CONNECTIONS_DB_PROJECT + - _APP_CONNECTIONS_CACHE + - _APP_CONNECTIONS_QUEUE - *x-env-storage - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG @@ -319,12 +351,19 @@ services: environment: - _APP_ENV - _APP_OPENSSL_KEY_V1 + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS - _APP_REDIS_HOST - _APP_REDIS_PORT - _APP_REDIS_USER - _APP_REDIS_PASS - - _APP_DB_CONSOLE - - _APP_DB_PROJECT + - _APP_CONNECTIONS_DB_CONSOLE + - _APP_CONNECTIONS_DB_PROJECT + - _APP_CONNECTIONS_CACHE + - _APP_CONNECTIONS_QUEUE - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG @@ -346,12 +385,19 @@ services: - _APP_OPENSSL_KEY_V1 - _APP_EXECUTOR_SECRET - _APP_EXECUTOR_HOST + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS - _APP_REDIS_HOST - _APP_REDIS_PORT - _APP_REDIS_USER - _APP_REDIS_PASS - - _APP_DB_CONSOLE - - _APP_DB_PROJECT + - _APP_CONNECTIONS_DB_CONSOLE + - _APP_CONNECTIONS_DB_PROJECT + - _APP_CONNECTIONS_CACHE + - _APP_CONNECTIONS_QUEUE - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG @@ -376,12 +422,19 @@ services: - _APP_DOMAIN - _APP_DOMAIN_TARGET - _APP_SYSTEM_SECURITY_EMAIL_ADDRESS + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS - _APP_REDIS_HOST - _APP_REDIS_PORT - _APP_REDIS_USER - _APP_REDIS_PASS - - _APP_DB_CONSOLE - - _APP_DB_PROJECT + - _APP_CONNECTIONS_DB_CONSOLE + - _APP_CONNECTIONS_DB_PROJECT + - _APP_CONNECTIONS_CACHE + - _APP_CONNECTIONS_QUEUE - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG @@ -402,12 +455,19 @@ services: environment: - _APP_ENV - _APP_OPENSSL_KEY_V1 + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS - _APP_REDIS_HOST - _APP_REDIS_PORT - _APP_REDIS_USER - _APP_REDIS_PASS - - _APP_DB_CONSOLE - - _APP_DB_PROJECT + - _APP_CONNECTIONS_DB_CONSOLE + - _APP_CONNECTIONS_DB_PROJECT + - _APP_CONNECTIONS_CACHE + - _APP_CONNECTIONS_QUEUE - _APP_FUNCTIONS_TIMEOUT - _APP_EXECUTOR_SECRET - _APP_EXECUTOR_HOST @@ -479,6 +539,7 @@ services: - _APP_REDIS_PORT - _APP_REDIS_USER - _APP_REDIS_PASS + - _APP_CONNECTIONS_QUEUE - _APP_SMTP_HOST - _APP_SMTP_PORT - _APP_SMTP_SECURE @@ -505,6 +566,7 @@ services: - _APP_REDIS_PORT - _APP_REDIS_USER - _APP_REDIS_PASS + - _APP_CONNECTIONS_QUEUE - _APP_SMS_PROVIDER - _APP_SMS_FROM - _APP_LOGGING_PROVIDER @@ -528,12 +590,18 @@ services: - _APP_DOMAIN - _APP_DOMAIN_TARGET - _APP_OPENSSL_KEY_V1 + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS - _APP_REDIS_HOST - _APP_REDIS_PORT - _APP_REDIS_USER - _APP_REDIS_PASS - - _APP_DB_CONSOLE - - _APP_DB_PROJECT + - _APP_CONNECTIONS_DB_CONSOLE + - _APP_CONNECTIONS_DB_PROJECT + - _APP_CONNECTIONS_CACHE - _APP_MAINTENANCE_INTERVAL - _APP_MAINTENANCE_RETENTION_EXECUTION - _APP_MAINTENANCE_RETENTION_CACHE @@ -559,16 +627,22 @@ services: environment: - _APP_ENV - _APP_OPENSSL_KEY_V1 - - _APP_DB_CONSOLE - - _APP_DB_PROJECT - - _APP_INFLUXDB_HOST - - _APP_INFLUXDB_PORT - - _APP_USAGE_TIMESERIES_INTERVAL - - _APP_USAGE_DATABASE_INTERVAL + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS - _APP_REDIS_HOST - _APP_REDIS_PORT - _APP_REDIS_USER - _APP_REDIS_PASS + - _APP_INFLUXDB_HOST + - _APP_INFLUXDB_PORT + - _APP_CONNECTIONS_DB_CONSOLE + - _APP_CONNECTIONS_DB_PROJECT + - _APP_CONNECTIONS_CACHE + - _APP_USAGE_TIMESERIES_INTERVAL + - _APP_USAGE_DATABASE_INTERVAL - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG @@ -591,16 +665,22 @@ services: environment: - _APP_ENV - _APP_OPENSSL_KEY_V1 - - _APP_DB_CONSOLE - - _APP_DB_PROJECT - - _APP_INFLUXDB_HOST - - _APP_INFLUXDB_PORT - - _APP_USAGE_TIMESERIES_INTERVAL - - _APP_USAGE_DATABASE_INTERVAL + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS - _APP_REDIS_HOST - _APP_REDIS_PORT - _APP_REDIS_USER - _APP_REDIS_PASS + - _APP_INFLUXDB_HOST + - _APP_INFLUXDB_PORT + - _APP_CONNECTIONS_DB_CONSOLE + - _APP_CONNECTIONS_DB_PROJECT + - _APP_CONNECTIONS_CACHE + - _APP_USAGE_TIMESERIES_INTERVAL + - _APP_USAGE_DATABASE_INTERVAL - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG @@ -622,6 +702,7 @@ services: - _APP_REDIS_PORT - _APP_REDIS_USER - _APP_REDIS_PASS + - _APP_CONNECTIONS_QUEUE mariadb: image: mariadb:10.7 # fix issues when upgrading using: mysql_upgrade -u root -p @@ -644,7 +725,6 @@ services: # smtp: # image: appwrite/smtp:1.2.0 # container_name: appwrite-smtp - # restart: unless-stopped # networks: # - appwrite # environment: @@ -731,7 +811,6 @@ services: image: adminer container_name: appwrite-adminer <<: *x-logging - restart: always ports: - 9506:8080 networks: @@ -739,7 +818,6 @@ services: # redis-commander: # image: rediscommander/redis-commander:latest - # restart: unless-stopped # networks: # - appwrite # environment: @@ -749,7 +827,6 @@ services: # resque: # image: appwrite/resque-web:1.1.0 - # restart: unless-stopped # networks: # - appwrite # ports: @@ -763,7 +840,6 @@ services: # chronograf: # image: chronograf:1.6 # container_name: appwrite-chronograf - # restart: unless-stopped # networks: # - appwrite # volumes: diff --git a/docs/references/health/get-cache.md b/docs/references/health/get-cache.md index 91abcd6bc5..632c02208d 100644 --- a/docs/references/health/get-cache.md +++ b/docs/references/health/get-cache.md @@ -1 +1 @@ -Check the Appwrite in-memory cache server is up and connection is successful. \ No newline at end of file +Check the Appwrite in-memory cache servers are up and connection is successful. \ No newline at end of file diff --git a/docs/references/health/get-db.md b/docs/references/health/get-db.md index 9652d0d3e3..7381e51f70 100644 --- a/docs/references/health/get-db.md +++ b/docs/references/health/get-db.md @@ -1 +1 @@ -Check the Appwrite database server is up and connection is successful. \ No newline at end of file +Check the Appwrite database servers are up and connection is successful. \ No newline at end of file diff --git a/docs/references/health/get-pubsub.md b/docs/references/health/get-pubsub.md new file mode 100644 index 0000000000..8f86411e8f --- /dev/null +++ b/docs/references/health/get-pubsub.md @@ -0,0 +1 @@ +Check the Appwrite pub-sub servers are up and connection is successful. \ No newline at end of file diff --git a/docs/references/health/get-queue.md b/docs/references/health/get-queue.md new file mode 100644 index 0000000000..e4558f941f --- /dev/null +++ b/docs/references/health/get-queue.md @@ -0,0 +1 @@ +Check the Appwrite queue messaging servers are up and connection is successful. \ No newline at end of file diff --git a/src/Appwrite/Database/DatabasePool.php b/src/Appwrite/Database/DatabasePool.php deleted file mode 100644 index ebd6a883c9..0000000000 --- a/src/Appwrite/Database/DatabasePool.php +++ /dev/null @@ -1,221 +0,0 @@ -consoleDB = array_key_first($consoleDB); - $this->dsns = array_merge($consoleDB, $projectDB); - - /** Create PDO pool instances for all the dsns */ - foreach ($this->dsns as $name => $dsn) { - $dsn = new DSN($dsn); - $pdoConfig = (new PDOConfig()) - ->withHost($dsn->getHost()) - ->withPort($dsn->getPort()) - ->withDbName($dsn->getDatabase()) - ->withCharset('utf8mb4') - ->withUsername($dsn->getUser()) - ->withPassword($dsn->getPassword()) - ->withOptions([ - PDO::ATTR_ERRMODE => App::isDevelopment() ? PDO::ERRMODE_WARNING : PDO::ERRMODE_SILENT, // If in production mode, warnings are not displayed - PDO::ATTR_TIMEOUT => 3, // Seconds - PDO::ATTR_PERSISTENT => true, - PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC, - PDO::ATTR_EMULATE_PREPARES => true, - PDO::ATTR_STRINGIFY_FETCHES => true - ]); - - $pool = new PDOPool($pdoConfig, $name, 64); - - $this->pools[$name] = $pool; - } - } - - /** - * Get a single PDO instance by database name - * - * @param string $name - * - * @return ?PDO - */ - public function getPDO(string $name): ?PDO - { - $dsn = $this->dsns[$name] ?? throw new Exception("Database with name : $name not found.", 500); - - $dsn = new DSN($dsn); - $dbHost = $dsn->getHost(); - $dbPort = $dsn->getPort(); - $dbUser = $dsn->getUser(); - $dbPass = $dsn->getPassword(); - $dbScheme = $dsn->getDatabase(); - - $pdo = new PDO("mysql:host={$dbHost};port={$dbPort};dbname={$dbScheme};charset=utf8mb4", $dbUser, $dbPass, array( - PDO::ATTR_TIMEOUT => 3, // Seconds - PDO::ATTR_PERSISTENT => true, - PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC, - PDO::ATTR_ERRMODE => App::isDevelopment() ? PDO::ERRMODE_WARNING : PDO::ERRMODE_SILENT, // If in production mode, warnings are not displayed - PDO::ATTR_EMULATE_PREPARES => true, - PDO::ATTR_STRINGIFY_FETCHES => true - )); - - return $pdo; - } - - /** - * Get a PDO instance from the list of available database pools. Meant to be used in co-routines - * - * @param string $projectId - * - * @return array - */ - public function getPDOFromPool(string $name): PDOWrapper - { - $pool = $this->pools[$name] ?? throw new Exception("Database pool with name : $name not found. Check the value of _APP_DB_PROJECT in .env", 500); - $pdo = $pool->get(); - return $pdo; - } - - /** - * Get a random PDO instance from the available database pools - * - * @return PDOWrapper - */ - public function getAnyFromPool(): PDOWrapper - { - $name = array_rand($this->pools); - $pool = $this->pools[$name] ?? throw new Exception("Database pool with name : $name not found. Check the value of _APP_DB_PROJECT in .env", 500); - $pdo = $pool->get(); - return $pdo; - } - - public function reset(): void - { - foreach ($this->pools as $pool) { - $pool->reset(); - } - } - - /** - * Return a PDO instance back to its database pool - * - * @param PDOProxy $db - * @param string $name - * - * @return void - */ - public function put(PDOProxy $db, string $name): void - { - $pool = $this->pools[$name] ?? null; - if ($pool === null) { - throw new Exception("Failed to put PDO into database pool. Database pool with name : $name not found", 500); - } - $pool->put($db); - } - - /** - * Get the name of the console DB - * - * @return ?string - */ - public function getConsoleDB(): ?string - { - if (empty($this->consoleDB)) { - throw new Exception('Console DB is not defined', 500); - }; - - return $this->consoleDB; - } - - public static function wait(Database $database, string $collection) - { - $attempts = 0; - do { - try { - $attempts++; - if (!$database->exists($database->getDefaultDatabase(), $collection)) { - throw new Exception('Collection not ready'); - } - break; // leave loop if successful - } catch (\Exception $e) { - Console::warning("Database not ready. Retrying connection ({$attempts})..."); - if ($attempts >= DATABASE_RECONNECT_MAX_ATTEMPTS) { - throw new \Exception('Failed to connect to database: ' . $e->getMessage()); - } - sleep(DATABASE_RECONNECT_SLEEP); - } - } while ($attempts < DATABASE_RECONNECT_MAX_ATTEMPTS); - - return $database; - } - - /** - * Get a database instance from a PDO and cache - * - * @param PDO|PDOProxy $pdo - * @param \Redis $redis - * @param string $namespace - * - * @return Database - */ - public static function getDatabase(PDO|PDOProxy $pdo, \Redis $redis, string $namespace = ''): Database - { - $cache = new Cache(new RedisCache($redis)); - $database = new Database(new MariaDB($pdo), $cache); - $database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); - $database->setNamespace($namespace); - return $database; - } -} diff --git a/src/Appwrite/Database/PDOPool.php b/src/Appwrite/Database/PDOPool.php deleted file mode 100644 index a3db40f754..0000000000 --- a/src/Appwrite/Database/PDOPool.php +++ /dev/null @@ -1,47 +0,0 @@ -pool = new SwoolePDOPool($pdoConfig, $size); - $this->name = $name; - } - - public function getActiveConnections() - { - return $this->activeConnections; - } - - public function get(float $timeout = -1): PDOWrapper - { - $pdo = $this->pool->get($timeout); - $this->activeConnections[] = $pdo; - return new PDOWrapper($pdo, $this->name); - } - - public function put(PDOWrapper $pdo): void - { - $this->pool->put($pdo->getConnection()); - unset($this->activeConnections[array_search($pdo, $this->activeConnections)]); - } - - public function reset(): void - { - foreach ($this->activeConnections as $connection) { - $this->pool->put($connection); - } - $this->activeConnections = []; - } -} diff --git a/src/Appwrite/Database/PDOWrapper.php b/src/Appwrite/Database/PDOWrapper.php deleted file mode 100644 index 7e2b2b7b6f..0000000000 --- a/src/Appwrite/Database/PDOWrapper.php +++ /dev/null @@ -1,27 +0,0 @@ -connection = $connection; - $this->name = $name; - } - - public function getName() - { - return $this->name; - } - - public function getConnection() - { - return $this->connection; - } -} diff --git a/src/Appwrite/Extend/PDO.php b/src/Appwrite/Extend/PDO.php deleted file mode 100644 index bc3f42afb9..0000000000 --- a/src/Appwrite/Extend/PDO.php +++ /dev/null @@ -1,110 +0,0 @@ -dsn = $dsn; - $this->username = $username; - $this->passwd = $passwd; - $this->options = $options; - - $this->pdo = new PDONative($dsn, $username, $passwd, $options); - } - - public function setAttribute($attribute, $value) - { - return $this->pdo->setAttribute($attribute, $value); - } - - public function prepare($statement, $driver_options = null) - { - return new PDOStatement($this, $this->pdo->prepare($statement, [])); - } - - public function quote($string, $parameter_type = PDONative::PARAM_STR) - { - return $this->pdo->quote($string, $parameter_type); - } - - public function beginTransaction() - { - try { - $result = $this->pdo->beginTransaction(); - } catch (\Throwable $th) { - $this->pdo = $this->reconnect(); - $result = $this->pdo->beginTransaction(); - } - - return $result; - } - - public function rollBack() - { - try { - $result = $this->pdo->rollBack(); - } catch (\Throwable $th) { - $this->pdo = $this->reconnect(); - return false; - } - - return $result; - } - - public function commit() - { - try { - $result = $this->pdo->commit(); - } catch (\Throwable $th) { - $this->pdo = $this->reconnect(); - $result = $this->pdo->commit(); - } - - return $result; - } - - public function reconnect(): PDONative - { - $this->pdo = new PDONative($this->dsn, $this->username, $this->passwd, $this->options); - - echo '[PDO] MySQL connection restarted' . PHP_EOL; - - // Connection settings - $this->pdo->setAttribute(PDONative::ATTR_DEFAULT_FETCH_MODE, PDONative::FETCH_ASSOC); // Return arrays - $this->pdo->setAttribute(PDONative::ATTR_ERRMODE, PDONative::ERRMODE_EXCEPTION); // Handle all errors with exceptions - - return $this->pdo; - } -} diff --git a/src/Appwrite/Extend/PDOStatement.php b/src/Appwrite/Extend/PDOStatement.php deleted file mode 100644 index 9c5a83ec34..0000000000 --- a/src/Appwrite/Extend/PDOStatement.php +++ /dev/null @@ -1,115 +0,0 @@ -pdo = &$pdo; - $this->PDOStatement = $PDOStatement; - } - - public function bindValue($parameter, $value, $data_type = PDONative::PARAM_STR) - { - $this->values[$parameter] = ['value' => $value, 'data_type' => $data_type]; - - $result = $this->PDOStatement->bindValue($parameter, $value, $data_type); - - return $result; - } - - public function bindParam($parameter, &$variable, $data_type = PDONative::PARAM_STR, $length = null, $driver_options = null) - { - $this->params[$parameter] = ['value' => &$variable, 'data_type' => $data_type, 'length' => $length, 'driver_options' => $driver_options]; - - $result = $this->PDOStatement->bindParam($parameter, $variable, $data_type, $length, $driver_options); - - return $result; - } - - public function bindColumn($column, &$param, $type = null, $maxlen = null, $driverdata = null) - { - $this->columns[$column] = ['param' => &$param, 'type' => $type, 'maxlen' => $maxlen, 'driverdata' => $driverdata]; - - $result = $this->PDOStatement->bindColumn($column, $param, $type, $maxlen, $driverdata); - - return $result; - } - - public function execute($input_parameters = null) - { - try { - $result = $this->PDOStatement->execute($input_parameters); - } catch (\Throwable $th) { - $this->pdo = $this->pdo->reconnect(); - $this->PDOStatement = $this->pdo->prepare($this->PDOStatement->queryString, []); - - foreach ($this->values as $key => $set) { - $this->PDOStatement->bindValue($key, $set['value'], $set['data_type']); - } - - foreach ($this->params as $key => $set) { - $this->PDOStatement->bindParam($key, $set['variable'], $set['data_type'], $set['length'], $set['driver_options']); - } - - foreach ($this->columns as $key => $set) { - $this->PDOStatement->bindColumn($key, $set['param'], $set['type'], $set['maxlen'], $set['driverdata']); - } - - $result = $this->PDOStatement->execute($input_parameters); - } - - return $result; - } - - public function fetch($fetch_style = PDONative::FETCH_ASSOC, $cursor_orientation = PDONative::FETCH_ORI_NEXT, $cursor_offset = 0) - { - $result = $this->PDOStatement->fetch($fetch_style, $cursor_orientation, $cursor_offset); - - return $result; - } - - /** - * Fetch All - * - * @param int $fetch_style - * @param mixed $fetch_args - * - * @return array|false - */ - public function fetchAll(int $fetch_style = PDO::FETCH_BOTH, mixed ...$fetch_args) - { - $result = $this->PDOStatement->fetchAll(); - - return $result; - } -} diff --git a/src/Appwrite/Resque/Worker.php b/src/Appwrite/Resque/Worker.php index c1ef264670..5d05d77576 100644 --- a/src/Appwrite/Resque/Worker.php +++ b/src/Appwrite/Resque/Worker.php @@ -2,8 +2,11 @@ namespace Appwrite\Resque; -use Appwrite\Database\DatabasePool; +use Exception; use Utopia\App; +use Utopia\Cache\Cache; +use Utopia\Config\Config; +use Utopia\Cache\Adapter\Sharding; use Utopia\Database\Database; use Utopia\Storage\Device; use Utopia\Storage\Storage; @@ -13,7 +16,6 @@ use Utopia\Storage\Device\Linode; use Utopia\Storage\Device\Wasabi; use Utopia\Storage\Device\Backblaze; use Utopia\Storage\Device\S3; -use Exception; use Utopia\Database\Document; use Utopia\Database\Validator\Authorization; @@ -134,7 +136,12 @@ abstract class Worker */ public function tearDown(): void { + global $register; + try { + $pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */ + $pools->reclaim(); + $this->shutdown(); } catch (\Throwable $error) { foreach (self::$errorCallbacks as $errorCallback) { @@ -165,22 +172,23 @@ abstract class Worker protected function getProjectDB(Document $project): Database { global $register; - $database = $project->getAttribute('database', ''); - $internalId = $project->getInternalId(); - if (empty($database)) { - throw new \Exception('Database name not provided - cannot get database'); + + $pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */ + + if ($project->isEmpty() || $project->getId() === 'console') { + return $this->getConsoleDB(); } - $cache = $register->get('cache'); - $dbPool = $register->get('dbPool'); - $namespace = "_$internalId"; - $pdo = $dbPool->getPDO($database); - $dbForProject = DatabasePool::wait( - DatabasePool::getDatabase($pdo, $cache, $namespace), - 'projects' - ); + $dbAdapter = $pools + ->get($project->getAttribute('database')) + ->pop() + ->getResource() + ; - return $dbForProject; + $database = new Database($dbAdapter, $this->getCache()); + $database->setNamespace('_' . $project->getInternalId()); + + return $database; } /** @@ -190,21 +198,45 @@ abstract class Worker protected function getConsoleDB(): Database { global $register; - $cache = $register->get('cache'); - $dbPool = $register->get('dbPool'); - $database = $dbPool->getConsoleDB(); - if (empty($database)) { - throw new \Exception('Database name not provided - cannot get database'); + + $pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */ + + $dbAdapter = $pools + ->get('console') + ->pop() + ->getResource() + ; + + $database = new Database($dbAdapter, $this->getCache()); + + $database->setNamespace('console'); + + return $database; + } + + + /** + * Get Cache + * @return Cache + */ + protected function getCache(): Cache + { + global $register; + + $pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */ + + $list = Config::getParam('pools-cache', []); + $adapters = []; + + foreach ($list as $value) { + $adapters[] = $pools + ->get($value) + ->pop() + ->getResource() + ; } - $namespace = "_console"; - $pdo = $dbPool->getPDO($database); - $dbForConsole = DatabasePool::wait( - DatabasePool::getDatabase($pdo, $cache, $namespace), - '_metadata' - ); - - return $dbForConsole; + return new Cache(new Sharding($adapters)); } /** @@ -227,7 +259,6 @@ abstract class Worker return $this->getDevice(APP_STORAGE_UPLOADS . '/app-' . $projectId); } - /** * Get Builds Storage Device * @param string $projectId of the project diff --git a/src/Appwrite/Utopia/Response.php b/src/Appwrite/Utopia/Response.php index e23335365a..07134c6ea9 100644 --- a/src/Appwrite/Utopia/Response.php +++ b/src/Appwrite/Utopia/Response.php @@ -207,6 +207,7 @@ class Response extends SwooleResponse public const MODEL_HEALTH_QUEUE = 'healthQueue'; public const MODEL_HEALTH_TIME = 'healthTime'; public const MODEL_HEALTH_ANTIVIRUS = 'healthAntivirus'; + public const MODEL_HEALTH_STATUS_LIST = 'healthStatusList'; // Deprecated public const MODEL_PERMISSIONS = 'permissions'; @@ -268,6 +269,7 @@ class Response extends SwooleResponse ->setModel(new BaseList('Phones List', self::MODEL_PHONE_LIST, 'phones', self::MODEL_PHONE)) ->setModel(new BaseList('Metric List', self::MODEL_METRIC_LIST, 'metrics', self::MODEL_METRIC, true, false)) ->setModel(new BaseList('Variables List', self::MODEL_VARIABLE_LIST, 'variables', self::MODEL_VARIABLE)) + ->setModel(new BaseList('Status List', self::MODEL_HEALTH_STATUS_LIST, 'statuses', self::MODEL_HEALTH_STATUS)) // Entities ->setModel(new Database()) ->setModel(new Collection()) diff --git a/src/Appwrite/Utopia/Response/Model/HealthStatus.php b/src/Appwrite/Utopia/Response/Model/HealthStatus.php index 23756de131..ba340107ac 100644 --- a/src/Appwrite/Utopia/Response/Model/HealthStatus.php +++ b/src/Appwrite/Utopia/Response/Model/HealthStatus.php @@ -10,6 +10,12 @@ class HealthStatus extends Model public function __construct() { $this + ->addRule('name', [ + 'type' => self::TYPE_STRING, + 'description' => 'Name of the service.', + 'default' => '', + 'example' => 'database', + ]) ->addRule('ping', [ 'type' => self::TYPE_INTEGER, 'description' => 'Duration in milliseconds how long the health check took.', diff --git a/tests/e2e/Services/Account/AccountBase.php b/tests/e2e/Services/Account/AccountBase.php index 71205dc6a0..e8bf146316 100644 --- a/tests/e2e/Services/Account/AccountBase.php +++ b/tests/e2e/Services/Account/AccountBase.php @@ -317,137 +317,137 @@ trait AccountBase return $data; } - // /** - // * @depends testCreateAccountSession - // */ - // public function testGetAccountLogs($data): array - // { - // sleep(10); - // $session = $data['session'] ?? ''; - // $sessionId = $data['sessionId'] ?? ''; - // $userId = $data['id'] ?? ''; - // /** - // * Test for SUCCESS - // */ - // $response = $this->client->call(Client::METHOD_GET, '/account/logs', array_merge([ - // 'origin' => 'http://localhost', - // 'content-type' => 'application/json', - // 'x-appwrite-project' => $this->getProject()['$id'], - // 'cookie' => 'a_session_' . $this->getProject()['$id'] . '=' . $session, - // ])); + /** + * @depends testCreateAccountSession + */ + public function testGetAccountLogs($data): array + { + sleep(10); + $session = $data['session'] ?? ''; + $sessionId = $data['sessionId'] ?? ''; + $userId = $data['id'] ?? ''; + /** + * Test for SUCCESS + */ + $response = $this->client->call(Client::METHOD_GET, '/account/logs', array_merge([ + 'origin' => 'http://localhost', + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'cookie' => 'a_session_' . $this->getProject()['$id'] . '=' . $session, + ])); - // $this->assertEquals($response['headers']['status-code'], 200); - // $this->assertIsArray($response['body']['logs']); - // $this->assertNotEmpty($response['body']['logs']); - // $this->assertCount(3, $response['body']['logs']); - // $this->assertIsNumeric($response['body']['total']); - // $this->assertContains($response['body']['logs'][1]['event'], ["session.create"]); - // $this->assertEquals($response['body']['logs'][1]['ip'], filter_var($response['body']['logs'][1]['ip'], FILTER_VALIDATE_IP)); - // $this->assertEquals(true, DateTime::isValid($response['body']['logs'][1]['time'])); + $this->assertEquals($response['headers']['status-code'], 200); + $this->assertIsArray($response['body']['logs']); + $this->assertNotEmpty($response['body']['logs']); + $this->assertCount(3, $response['body']['logs']); + $this->assertIsNumeric($response['body']['total']); + $this->assertContains($response['body']['logs'][1]['event'], ["session.create"]); + $this->assertEquals($response['body']['logs'][1]['ip'], filter_var($response['body']['logs'][1]['ip'], FILTER_VALIDATE_IP)); + $this->assertEquals(true, DateTime::isValid($response['body']['logs'][1]['time'])); - // $this->assertEquals('Windows', $response['body']['logs'][1]['osName']); - // $this->assertEquals('WIN', $response['body']['logs'][1]['osCode']); - // $this->assertEquals('10', $response['body']['logs'][1]['osVersion']); + $this->assertEquals('Windows', $response['body']['logs'][1]['osName']); + $this->assertEquals('WIN', $response['body']['logs'][1]['osCode']); + $this->assertEquals('10', $response['body']['logs'][1]['osVersion']); - // $this->assertEquals('browser', $response['body']['logs'][1]['clientType']); - // $this->assertEquals('Chrome', $response['body']['logs'][1]['clientName']); - // $this->assertEquals('CH', $response['body']['logs'][1]['clientCode']); - // $this->assertEquals('70.0', $response['body']['logs'][1]['clientVersion']); - // $this->assertEquals('Blink', $response['body']['logs'][1]['clientEngine']); + $this->assertEquals('browser', $response['body']['logs'][1]['clientType']); + $this->assertEquals('Chrome', $response['body']['logs'][1]['clientName']); + $this->assertEquals('CH', $response['body']['logs'][1]['clientCode']); + $this->assertEquals('70.0', $response['body']['logs'][1]['clientVersion']); + $this->assertEquals('Blink', $response['body']['logs'][1]['clientEngine']); - // $this->assertEquals('desktop', $response['body']['logs'][1]['deviceName']); - // $this->assertEquals('', $response['body']['logs'][1]['deviceBrand']); - // $this->assertEquals('', $response['body']['logs'][1]['deviceModel']); - // $this->assertEquals($response['body']['logs'][1]['ip'], filter_var($response['body']['logs'][1]['ip'], FILTER_VALIDATE_IP)); + $this->assertEquals('desktop', $response['body']['logs'][1]['deviceName']); + $this->assertEquals('', $response['body']['logs'][1]['deviceBrand']); + $this->assertEquals('', $response['body']['logs'][1]['deviceModel']); + $this->assertEquals($response['body']['logs'][1]['ip'], filter_var($response['body']['logs'][1]['ip'], FILTER_VALIDATE_IP)); - // $this->assertEquals('--', $response['body']['logs'][1]['countryCode']); - // $this->assertEquals('Unknown', $response['body']['logs'][1]['countryName']); + $this->assertEquals('--', $response['body']['logs'][1]['countryCode']); + $this->assertEquals('Unknown', $response['body']['logs'][1]['countryName']); - // $this->assertContains($response['body']['logs'][2]['event'], ["user.create"]); - // $this->assertEquals($response['body']['logs'][2]['ip'], filter_var($response['body']['logs'][2]['ip'], FILTER_VALIDATE_IP)); - // $this->assertEquals(true, DateTime::isValid($response['body']['logs'][2]['time'])); + $this->assertContains($response['body']['logs'][2]['event'], ["user.create"]); + $this->assertEquals($response['body']['logs'][2]['ip'], filter_var($response['body']['logs'][2]['ip'], FILTER_VALIDATE_IP)); + $this->assertEquals(true, DateTime::isValid($response['body']['logs'][2]['time'])); - // $this->assertEquals('Windows', $response['body']['logs'][2]['osName']); - // $this->assertEquals('WIN', $response['body']['logs'][2]['osCode']); - // $this->assertEquals('10', $response['body']['logs'][2]['osVersion']); + $this->assertEquals('Windows', $response['body']['logs'][2]['osName']); + $this->assertEquals('WIN', $response['body']['logs'][2]['osCode']); + $this->assertEquals('10', $response['body']['logs'][2]['osVersion']); - // $this->assertEquals('browser', $response['body']['logs'][2]['clientType']); - // $this->assertEquals('Chrome', $response['body']['logs'][2]['clientName']); - // $this->assertEquals('CH', $response['body']['logs'][2]['clientCode']); - // $this->assertEquals('70.0', $response['body']['logs'][2]['clientVersion']); - // $this->assertEquals('Blink', $response['body']['logs'][2]['clientEngine']); + $this->assertEquals('browser', $response['body']['logs'][2]['clientType']); + $this->assertEquals('Chrome', $response['body']['logs'][2]['clientName']); + $this->assertEquals('CH', $response['body']['logs'][2]['clientCode']); + $this->assertEquals('70.0', $response['body']['logs'][2]['clientVersion']); + $this->assertEquals('Blink', $response['body']['logs'][2]['clientEngine']); - // $this->assertEquals('desktop', $response['body']['logs'][2]['deviceName']); - // $this->assertEquals('', $response['body']['logs'][2]['deviceBrand']); - // $this->assertEquals('', $response['body']['logs'][2]['deviceModel']); - // $this->assertEquals($response['body']['logs'][2]['ip'], filter_var($response['body']['logs'][2]['ip'], FILTER_VALIDATE_IP)); + $this->assertEquals('desktop', $response['body']['logs'][2]['deviceName']); + $this->assertEquals('', $response['body']['logs'][2]['deviceBrand']); + $this->assertEquals('', $response['body']['logs'][2]['deviceModel']); + $this->assertEquals($response['body']['logs'][2]['ip'], filter_var($response['body']['logs'][2]['ip'], FILTER_VALIDATE_IP)); - // $this->assertEquals('--', $response['body']['logs'][2]['countryCode']); - // $this->assertEquals('Unknown', $response['body']['logs'][2]['countryName']); + $this->assertEquals('--', $response['body']['logs'][2]['countryCode']); + $this->assertEquals('Unknown', $response['body']['logs'][2]['countryName']); - // $responseLimit = $this->client->call(Client::METHOD_GET, '/account/logs', array_merge([ - // 'origin' => 'http://localhost', - // 'content-type' => 'application/json', - // 'x-appwrite-project' => $this->getProject()['$id'], - // 'cookie' => 'a_session_' . $this->getProject()['$id'] . '=' . $session, - // ]), [ - // 'queries' => [ 'limit(1)' ], - // ]); + $responseLimit = $this->client->call(Client::METHOD_GET, '/account/logs', array_merge([ + 'origin' => 'http://localhost', + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'cookie' => 'a_session_' . $this->getProject()['$id'] . '=' . $session, + ]), [ + 'queries' => [ 'limit(1)' ], + ]); - // $this->assertEquals($responseLimit['headers']['status-code'], 200); - // $this->assertIsArray($responseLimit['body']['logs']); - // $this->assertNotEmpty($responseLimit['body']['logs']); - // $this->assertCount(1, $responseLimit['body']['logs']); - // $this->assertIsNumeric($responseLimit['body']['total']); + $this->assertEquals($responseLimit['headers']['status-code'], 200); + $this->assertIsArray($responseLimit['body']['logs']); + $this->assertNotEmpty($responseLimit['body']['logs']); + $this->assertCount(1, $responseLimit['body']['logs']); + $this->assertIsNumeric($responseLimit['body']['total']); - // $this->assertEquals($response['body']['logs'][0], $responseLimit['body']['logs'][0]); + $this->assertEquals($response['body']['logs'][0], $responseLimit['body']['logs'][0]); - // $responseOffset = $this->client->call(Client::METHOD_GET, '/account/logs', array_merge([ - // 'origin' => 'http://localhost', - // 'content-type' => 'application/json', - // 'x-appwrite-project' => $this->getProject()['$id'], - // 'cookie' => 'a_session_' . $this->getProject()['$id'] . '=' . $session, - // ]), [ - // 'queries' => [ 'offset(1)' ], - // ]); + $responseOffset = $this->client->call(Client::METHOD_GET, '/account/logs', array_merge([ + 'origin' => 'http://localhost', + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'cookie' => 'a_session_' . $this->getProject()['$id'] . '=' . $session, + ]), [ + 'queries' => [ 'offset(1)' ], + ]); - // $this->assertEquals($responseOffset['headers']['status-code'], 200); - // $this->assertIsArray($responseOffset['body']['logs']); - // $this->assertNotEmpty($responseOffset['body']['logs']); - // $this->assertCount(2, $responseOffset['body']['logs']); - // $this->assertIsNumeric($responseOffset['body']['total']); + $this->assertEquals($responseOffset['headers']['status-code'], 200); + $this->assertIsArray($responseOffset['body']['logs']); + $this->assertNotEmpty($responseOffset['body']['logs']); + $this->assertCount(2, $responseOffset['body']['logs']); + $this->assertIsNumeric($responseOffset['body']['total']); - // $this->assertEquals($response['body']['logs'][1], $responseOffset['body']['logs'][0]); + $this->assertEquals($response['body']['logs'][1], $responseOffset['body']['logs'][0]); - // $responseLimitOffset = $this->client->call(Client::METHOD_GET, '/account/logs', array_merge([ - // 'origin' => 'http://localhost', - // 'content-type' => 'application/json', - // 'x-appwrite-project' => $this->getProject()['$id'], - // 'cookie' => 'a_session_' . $this->getProject()['$id'] . '=' . $session, - // ]), [ - // 'queries' => [ 'limit(1)', 'offset(1)' ], - // ]); + $responseLimitOffset = $this->client->call(Client::METHOD_GET, '/account/logs', array_merge([ + 'origin' => 'http://localhost', + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'cookie' => 'a_session_' . $this->getProject()['$id'] . '=' . $session, + ]), [ + 'queries' => [ 'limit(1)', 'offset(1)' ], + ]); - // $this->assertEquals($responseLimitOffset['headers']['status-code'], 200); - // $this->assertIsArray($responseLimitOffset['body']['logs']); - // $this->assertNotEmpty($responseLimitOffset['body']['logs']); - // $this->assertCount(1, $responseLimitOffset['body']['logs']); - // $this->assertIsNumeric($responseLimitOffset['body']['total']); + $this->assertEquals($responseLimitOffset['headers']['status-code'], 200); + $this->assertIsArray($responseLimitOffset['body']['logs']); + $this->assertNotEmpty($responseLimitOffset['body']['logs']); + $this->assertCount(1, $responseLimitOffset['body']['logs']); + $this->assertIsNumeric($responseLimitOffset['body']['total']); - // $this->assertEquals($response['body']['logs'][1], $responseLimitOffset['body']['logs'][0]); - // /** - // * Test for FAILURE - // */ - // $response = $this->client->call(Client::METHOD_GET, '/account/logs', array_merge([ - // 'origin' => 'http://localhost', - // 'content-type' => 'application/json', - // 'x-appwrite-project' => $this->getProject()['$id'], - // ])); + $this->assertEquals($response['body']['logs'][1], $responseLimitOffset['body']['logs'][0]); + /** + * Test for FAILURE + */ + $response = $this->client->call(Client::METHOD_GET, '/account/logs', array_merge([ + 'origin' => 'http://localhost', + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ])); - // $this->assertEquals($response['headers']['status-code'], 401); + $this->assertEquals($response['headers']['status-code'], 401); - // return $data; - // } + return $data; + } // TODO Add tests for OAuth2 session creation diff --git a/tests/e2e/Services/Health/HealthCustomServerTest.php b/tests/e2e/Services/Health/HealthCustomServerTest.php index 47a2268e21..96c9bde5c7 100644 --- a/tests/e2e/Services/Health/HealthCustomServerTest.php +++ b/tests/e2e/Services/Health/HealthCustomServerTest.php @@ -47,9 +47,9 @@ class HealthCustomServerTest extends Scope ], $this->getHeaders()), []); $this->assertEquals(200, $response['headers']['status-code']); - $this->assertEquals('pass', $response['body']['status']); - $this->assertIsInt($response['body']['ping']); - $this->assertLessThan(100, $response['body']['ping']); + $this->assertEquals('pass', $response['body']['statuses'][0]['status']); + $this->assertIsInt($response['body']['statuses'][0]['ping']); + $this->assertLessThan(100, $response['body']['statuses'][0]['ping']); /** * Test for FAILURE @@ -69,9 +69,53 @@ class HealthCustomServerTest extends Scope ], $this->getHeaders()), []); $this->assertEquals(200, $response['headers']['status-code']); - $this->assertEquals('pass', $response['body']['status']); - $this->assertIsInt($response['body']['ping']); - $this->assertLessThan(100, $response['body']['ping']); + $this->assertEquals('pass', $response['body']['statuses'][0]['status']); + $this->assertIsInt($response['body']['statuses'][0]['ping']); + $this->assertLessThan(100, $response['body']['statuses'][0]['ping']); + + /** + * Test for FAILURE + */ + + return []; + } + + public function testQueueSuccess(): array + { + /** + * Test for SUCCESS + */ + $response = $this->client->call(Client::METHOD_GET, '/health/queue', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), []); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals('pass', $response['body']['statuses'][0]['status']); + $this->assertIsInt($response['body']['statuses'][0]['ping']); + $this->assertLessThan(100, $response['body']['statuses'][0]['ping']); + + /** + * Test for FAILURE + */ + + return []; + } + + public function testPubSubSuccess(): array + { + /** + * Test for SUCCESS + */ + $response = $this->client->call(Client::METHOD_GET, '/health/pubsub', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), []); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals('pass', $response['body']['statuses'][0]['status']); + $this->assertIsInt($response['body']['statuses'][0]['ping']); + $this->assertLessThan(100, $response['body']['statuses'][0]['ping']); /** * Test for FAILURE