Merge remote-tracking branch 'origin/fix-realtime-connections' into feat-cloud-jake

2024-06-30 04:00:34 +12:00 · 2022-11-10 13:26:19 +13:00 · 2022-11-10 13:26:19 +13:00 · ee69a306a9
parent be18943fa4 b6839fa8f1
commit ee69a306a9
7 changed files with 182 additions and 65 deletions
--- a/.env
+++ b/.env
@ -23,6 +23,7 @@ _APP_DB_SCHEMA=appwrite
 _APP_DB_USER=user
 _APP_DB_PASS=password
 _APP_DB_ROOT_PASS=rootsecretpassword
+_APP_DB_MAX_CONNECTIONS=1001
 _APP_STORAGE_DEVICE=Local
 _APP_STORAGE_S3_ACCESS_KEY=
 _APP_STORAGE_S3_SECRET=
--- a/app/config/variables.php
+++ b/app/config/variables.php
@ -306,6 +306,15 @@ return [
                'question' => '',
                'filter' => 'password'
            ],
+            [
+                'name' => '_APP_DB_MAX_CONNECTIONS',
+                'description' => 'MariaDB server maximum connections.',
+                'introduction' => '1.2.0',
+                'default' => 1001,
+                'required' => false,
+                'question' => '',
+                'filter' => ''
+            ],
        ],
    ],
    [
--- a/app/init.php
+++ b/app/init.php
@ -104,6 +104,7 @@ const APP_DATABASE_ATTRIBUTE_URL = 'url';
 const APP_DATABASE_ATTRIBUTE_INT_RANGE = 'intRange';
 const APP_DATABASE_ATTRIBUTE_FLOAT_RANGE = 'floatRange';
 const APP_DATABASE_ATTRIBUTE_STRING_MAX_LENGTH = 1073741824; // 2^32 bits / 4 bits per char
+const APP_DATABASE_DEFAULT_POOL_SIZE = 64;
 const APP_STORAGE_UPLOADS = '/storage/uploads';
 const APP_STORAGE_FUNCTIONS = '/storage/functions';
 const APP_STORAGE_BUILDS = '/storage/builds';
@ -495,7 +496,7 @@ $register->set('logger', function () {
    $adapter = new $classname($providerConfig);
    return new Logger($adapter);
 });
-$register->set('dbPool', function () {
+$register->set('dbPool', function ($size = APP_DATABASE_DEFAULT_POOL_SIZE) {
 // Register DB connection
    $dbHost = App::getEnv('_APP_DB_HOST', '');
    $dbPort = App::getEnv('_APP_DB_PORT', '');
@ -519,12 +520,12 @@ $register->set('dbPool', function () {
            PDO::ATTR_EMULATE_PREPARES => true,
            PDO::ATTR_STRINGIFY_FETCHES => true,
        ]),
-        64
+        $size
    );

    return $pool;
 });
-$register->set('redisPool', function () {
+$register->set('redisPool', function ($size = APP_DATABASE_DEFAULT_POOL_SIZE) {
    $redisHost = App::getEnv('_APP_REDIS_HOST', '');
    $redisPort = App::getEnv('_APP_REDIS_PORT', '');
    $redisUser = App::getEnv('_APP_REDIS_USER', '');
@ -541,7 +542,7 @@ $register->set('redisPool', function () {
        ->withPort($redisPort)
        ->withAuth($redisAuth)
        ->withDbIndex(0),
-        64
+        $size
    );

    return $pool;
@ -1006,6 +1007,26 @@ function getDevice($root): Device
    }
 }

+/**
+ * Get database connection pool size for worker processes.
+ *
+ * @return int
+ * @throws \Exception
+ */
+function getWorkerPoolSize(): int
+{
+    $reservedConnections = APP_DATABASE_DEFAULT_POOL_SIZE; // Pool of default size is reserved for the HTTP API
+    $workerCount = swoole_cpu_num() * intval(App::getEnv('_APP_WORKER_PER_CORE', 6));
+    $maxConnections = App::getenv('_APP_DB_MAX_CONNECTIONS', 1001);
+    $workerConnections = $maxConnections - $reservedConnections;
+
+    if ($workerCount > $workerConnections) {
+        throw new \Exception('Worker pool size is too small. Increase the number of allowed database connections or decrease the number of workers.');
+    }
+
+    return (int)($workerConnections / $workerCount);
+}
+
 App::setResource('mode', function ($request) {
    /** @var Appwrite\Utopia\Request $request */

--- a/app/realtime.php
+++ b/app/realtime.php
@ -4,6 +4,7 @@ use Appwrite\Auth\Auth;
 use Appwrite\Messaging\Adapter\Realtime;
 use Appwrite\Network\Validator\Origin;
 use Appwrite\Utopia\Response;
+use Swoole\ConnectionPool;
 use Swoole\Http\Request as SwooleRequest;
 use Swoole\Http\Response as SwooleResponse;
 use Swoole\Runtime;
@ -35,6 +36,9 @@ Runtime::enableCoroutine(SWOOLE_HOOK_ALL);

 $realtime = new Realtime();

+$dbPool = $register->get('dbPool', args: [getWorkerPoolSize()]);
+$redisPool = $register->get('redisPool');
+
 /**
 * Table for statistics across all workers.
 */
@ -95,7 +99,7 @@ $logError = function (Throwable $error, string $action) use ($register) {

 $server->error($logError);

-function getDatabase(Registry &$register, string $namespace)
+function getDatabase(ConnectionPool $dbPool, ConnectionPool $redisPool, string $namespace)
 {
    $attempts = 0;

@ -103,8 +107,8 @@ function getDatabase(Registry &$register, string $namespace)
        try {
            $attempts++;

-            $db = $register->get('dbPool')->get();
-            $redis = $register->get('redisPool')->get();
+            $db = $dbPool->get();
+            $redis = $redisPool->get();

            $cache = new Cache(new RedisCache($redis));
            $database = new Database(new MySQL($db), $cache);
@ -127,23 +131,23 @@ function getDatabase(Registry &$register, string $namespace)

    return [
        $database,
-        function () use ($register, $db, $redis) {
-            $register->get('dbPool')->put($db);
-            $register->get('redisPool')->put($redis);
+        function () use ($dbPool, $redisPool, $db, $redis) {
+            $dbPool->put($db);
+            $redisPool->put($redis);
        }
    ];
 }

-$server->onStart(function () use ($stats, $register, $containerId, &$statsDocument, $logError) {
+$server->onStart(function () use ($stats, $dbPool, $redisPool, $containerId, &$statsDocument, $logError) {
    sleep(5); // wait for the initial database schema to be ready
    Console::success('Server started successfully');

    /**
     * Create document for this worker to share stats across Containers.
     */
-    go(function () use ($register, $containerId, &$statsDocument) {
+    go(function () use ($dbPool, $redisPool, $containerId, &$statsDocument) {
        $attempts = 0;
-        [$database, $returnDatabase] = getDatabase($register, '_console');
+        [$database, $returnDatabase] = getDatabase($dbPool, $redisPool, '_console');
        do {
            try {
                $attempts++;
@ -169,7 +173,7 @@ $server->onStart(function () use ($stats, $register, $containerId, &$statsDocume
    /**
     * Save current connections to the Database every 5 seconds.
     */
-    Timer::tick(5000, function () use ($register, $stats, &$statsDocument, $logError) {
+    Timer::tick(5000, function () use ($dbPool, $redisPool, $stats, &$statsDocument, $logError) {
        $payload = [];
        foreach ($stats as $projectId => $value) {
            $payload[$projectId] = $stats->get($projectId, 'connectionsTotal');
@ -179,7 +183,7 @@ $server->onStart(function () use ($stats, $register, $containerId, &$statsDocume
        }

        try {
-            [$database, $returnDatabase] = getDatabase($register, '_console');
+            [$database, $returnDatabase] = getDatabase($dbPool, $redisPool, '_console');

            $statsDocument
                ->setAttribute('timestamp', DateTime::now())
@ -194,18 +198,18 @@ $server->onStart(function () use ($stats, $register, $containerId, &$statsDocume
    });
 });

-$server->onWorkerStart(function (int $workerId) use ($server, $register, $stats, $realtime, $logError) {
+$server->onWorkerStart(function (int $workerId) use ($server, $dbPool, $redisPool, $stats, $realtime, $logError) {
    Console::success('Worker ' . $workerId . ' started successfully');

    $attempts = 0;
    $start = time();

-    Timer::tick(5000, function () use ($server, $register, $realtime, $stats, $logError) {
+    Timer::tick(5000, function () use ($server, $dbPool, $redisPool, $realtime, $stats, $logError) {
        /**
         * Sending current connections to project channels on the console project every 5 seconds.
         */
        if ($realtime->hasSubscriber('console', Role::users()->toString(), 'project')) {
-            [$database, $returnDatabase] = getDatabase($register, '_console');
+            [$database, $returnDatabase] = getDatabase($dbPool, $redisPool, '_console');

            $payload = [];

@ -286,7 +290,7 @@ $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats,
            $start = time();

            /** @var Redis $redis */
-            $redis = $register->get('redisPool')->get();
+            $redis = $redisPool->get();
            $redis->setOption(Redis::OPT_READ_TIMEOUT, -1);

            if ($redis->ping(true)) {
@ -296,7 +300,7 @@ $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats,
                Console::error('Pub/sub failed (worker: ' . $workerId . ')');
            }

-            $redis->subscribe(['realtime'], function (Redis $redis, string $channel, string $payload) use ($server, $workerId, $stats, $register, $realtime) {
+            $redis->subscribe(['realtime'], function (Redis $redis, string $channel, string $payload) use ($server, $workerId, $stats, $dbPool, $redisPool, $realtime) {
                $event = json_decode($payload, true);

                if ($event['permissionsChanged'] && isset($event['userId'])) {
@ -305,9 +309,9 @@ $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats,

                    if ($realtime->hasSubscriber($projectId, 'user:' . $userId)) {
                        $connection = array_key_first(reset($realtime->subscriptions[$projectId]['user:' . $userId]));
-                        [$consoleDatabase, $returnConsoleDatabase] = getDatabase($register, '_console');
+                        [$consoleDatabase, $returnConsoleDatabase] = getDatabase($dbPool, $redisPool, '_console');
                        $project = Authorization::skip(fn() => $consoleDatabase->getDocument('projects', $projectId));
-                        [$database, $returnDatabase] = getDatabase($register, "_{$project->getInternalId()}");
+                        [$database, $returnDatabase] = getDatabase($dbPool, $redisPool, "_{$project->getInternalId()}");

                        $user = $database->getDocument('users', $userId);

@ -344,25 +348,26 @@ $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats,
            call_user_func($logError, $th, "pubSubConnection");

            Console::error('Pub/sub error: ' . $th->getMessage());
-            $register->get('redisPool')->put($redis);
            $attempts++;
            sleep(DATABASE_RECONNECT_SLEEP);
            continue;
+        } finally {
+            $redisPool->put($redis);
        }
    }

    Console::error('Failed to restart pub/sub...');
 });

-$server->onOpen(function (int $connection, SwooleRequest $request) use ($server, $register, $stats, &$realtime, $logError) {
+$server->onOpen(function (int $connection, SwooleRequest $request) use ($server, $dbPool, $redisPool, $stats, &$realtime, $logError) {
    $app = new App('UTC');
    $request = new Request($request);
    $response = new Response(new SwooleResponse());

    /** @var PDO $db */
-    $db = $register->get('dbPool')->get();
+    $db = $dbPool->get();
    /** @var Redis $redis */
-    $redis = $register->get('redisPool')->get();
+    $redis = $redisPool->get();

    Console::info("Connection open (user: {$connection})");

@ -477,16 +482,16 @@ $server->onOpen(function (int $connection, SwooleRequest $request) use ($server,
        /**
         * Put used PDO and Redis Connections back into their pools.
         */
-        $register->get('dbPool')->put($db);
-        $register->get('redisPool')->put($redis);
+        $dbPool->put($db);
+        $redisPool->put($redis);
    }
 });

-$server->onMessage(function (int $connection, string $message) use ($server, $register, $realtime, $containerId) {
+$server->onMessage(function (int $connection, string $message) use ($server, $dbPool, $redisPool, $realtime, $containerId) {
    try {
        $response = new Response(new SwooleResponse());
-        $db = $register->get('dbPool')->get();
-        $redis = $register->get('redisPool')->get();
+        $db = $dbPool->get();
+        $redis = $redisPool->get();

        $cache = new Cache(new RedisCache($redis));
        $database = new Database(new MySQL($db), $cache);
@ -580,8 +585,8 @@ $server->onMessage(function (int $connection, string $message) use ($server, $re
            $server->close($connection, $th->getCode());
        }
    } finally {
-        $register->get('dbPool')->put($db);
-        $register->get('redisPool')->put($redis);
+        $dbPool->put($db);
+        $redisPool->put($redis);
    }
 });

--- a/composer.json
+++ b/composer.json
@ -53,7 +53,7 @@
        "utopia-php/config": "0.2.*",
        "utopia-php/database": "0.28.*",
        "utopia-php/locale": "0.4.*",
-        "utopia-php/registry": "0.5.*",
+        "utopia-php/registry": "dev-feat-allow-params as 0.5.0",
        "utopia-php/preloader": "0.2.*",
        "utopia-php/domains": "1.1.*",
        "utopia-php/swoole": "0.3.*",
--- a/composer.lock
+++ b/composer.lock
@ -4,7 +4,7 @@
        "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
        "This file is @generated automatically"
    ],
-    "content-hash": "51f81d435f4b5b7a9a6ea8f81b470353",
+    "content-hash": "6c9a067a247f45ee7c5c224577201d75",
    "packages": [
        {
            "name": "adhocore/jwt",
@ -115,15 +115,15 @@
        },
        {
            "name": "appwrite/php-runtimes",
-            "version": "0.11.0",
+            "version": "0.11.1",
            "source": {
                "type": "git",
                "url": "https://github.com/appwrite/runtimes.git",
-                "reference": "547fc026e11c0946846a8ac690898f5bf53be101"
+                "reference": "9d74a477ba3333cbcfac565c46fcf19606b7b603"
            },
            "require": {
                "php": ">=8.0",
-                "utopia-php/system": "0.4.*"
+                "utopia-php/system": "0.6.*"
            },
            "require-dev": {
                "phpunit/phpunit": "^9.3",
@ -154,7 +154,7 @@
                "php",
                "runtimes"
            ],
-            "time": "2022-08-15T14:03:36+00:00"
+            "time": "2022-11-07T16:45:52+00:00"
        },
        {
            "name": "chillerlan/php-qrcode",
@ -300,16 +300,16 @@
        },
        {
            "name": "colinmollenhour/credis",
-            "version": "v1.13.1",
+            "version": "v1.14.0",
            "source": {
                "type": "git",
                "url": "https://github.com/colinmollenhour/credis.git",
-                "reference": "85df015088e00daf8ce395189de22c8eb45c8d49"
+                "reference": "dccc8a46586475075fbb012d8bd523b8a938c2dc"
            },
            "dist": {
                "type": "zip",
-                "url": "https://api.github.com/repos/colinmollenhour/credis/zipball/85df015088e00daf8ce395189de22c8eb45c8d49",
-                "reference": "85df015088e00daf8ce395189de22c8eb45c8d49",
+                "url": "https://api.github.com/repos/colinmollenhour/credis/zipball/dccc8a46586475075fbb012d8bd523b8a938c2dc",
+                "reference": "dccc8a46586475075fbb012d8bd523b8a938c2dc",
                "shasum": ""
            },
            "require": {
@ -341,9 +341,9 @@
            "homepage": "https://github.com/colinmollenhour/credis",
            "support": {
                "issues": "https://github.com/colinmollenhour/credis/issues",
-                "source": "https://github.com/colinmollenhour/credis/tree/v1.13.1"
+                "source": "https://github.com/colinmollenhour/credis/tree/v1.14.0"
            },
-            "time": "2022-06-20T22:56:59+00:00"
+            "time": "2022-11-09T01:18:39+00:00"
        },
        {
            "name": "dragonmantank/cron-expression",
@ -803,6 +803,72 @@
            },
            "time": "2020-12-26T17:45:17+00:00"
        },
+        {
+            "name": "laravel/pint",
+            "version": "v1.2.0",
+            "source": {
+                "type": "git",
+                "url": "https://github.com/laravel/pint.git",
+                "reference": "1d276e4c803397a26cc337df908f55c2a4e90d86"
+            },
+            "dist": {
+                "type": "zip",
+                "url": "https://api.github.com/repos/laravel/pint/zipball/1d276e4c803397a26cc337df908f55c2a4e90d86",
+                "reference": "1d276e4c803397a26cc337df908f55c2a4e90d86",
+                "shasum": ""
+            },
+            "require": {
+                "ext-json": "*",
+                "ext-mbstring": "*",
+                "ext-tokenizer": "*",
+                "ext-xml": "*",
+                "php": "^8.0"
+            },
+            "require-dev": {
+                "friendsofphp/php-cs-fixer": "^3.11.0",
+                "illuminate/view": "^9.27",
+                "laravel-zero/framework": "^9.1.3",
+                "mockery/mockery": "^1.5.0",
+                "nunomaduro/larastan": "^2.2",
+                "nunomaduro/termwind": "^1.14.0",
+                "pestphp/pest": "^1.22.1"
+            },
+            "bin": [
+                "builds/pint"
+            ],
+            "type": "project",
+            "autoload": {
+                "psr-4": {
+                    "App\\": "app/",
+                    "Database\\Seeders\\": "database/seeders/",
+                    "Database\\Factories\\": "database/factories/"
+                }
+            },
+            "notification-url": "https://packagist.org/downloads/",
+            "license": [
+                "MIT"
+            ],
+            "authors": [
+                {
+                    "name": "Nuno Maduro",
+                    "email": "enunomaduro@gmail.com"
+                }
+            ],
+            "description": "An opinionated code formatter for PHP.",
+            "homepage": "https://laravel.com",
+            "keywords": [
+                "format",
+                "formatter",
+                "lint",
+                "linter",
+                "php"
+            ],
+            "support": {
+                "issues": "https://github.com/laravel/pint/issues",
+                "source": "https://github.com/laravel/pint"
+            },
+            "time": "2022-09-13T15:07:15+00:00"
+        },
        {
            "name": "matomo/device-detector",
            "version": "6.0.0",
@ -2205,16 +2271,16 @@
        },
        {
            "name": "utopia-php/registry",
-            "version": "0.5.0",
+            "version": "dev-feat-allow-params",
            "source": {
                "type": "git",
                "url": "https://github.com/utopia-php/registry.git",
-                "reference": "bedc4ed54527b2803e6dfdccc39449f98522b70d"
+                "reference": "6c571f8f4127094b3af8909d1b595fd6b937255d"
            },
            "dist": {
                "type": "zip",
-                "url": "https://api.github.com/repos/utopia-php/registry/zipball/bedc4ed54527b2803e6dfdccc39449f98522b70d",
-                "reference": "bedc4ed54527b2803e6dfdccc39449f98522b70d",
+                "url": "https://api.github.com/repos/utopia-php/registry/zipball/6c571f8f4127094b3af8909d1b595fd6b937255d",
+                "reference": "6c571f8f4127094b3af8909d1b595fd6b937255d",
                "shasum": ""
            },
            "require": {
@ -2251,9 +2317,9 @@
            ],
            "support": {
                "issues": "https://github.com/utopia-php/registry/issues",
-                "source": "https://github.com/utopia-php/registry/tree/0.5.0"
+                "source": "https://github.com/utopia-php/registry/tree/feat-allow-params"
            },
-            "time": "2021-03-10T10:45:22+00:00"
+            "time": "2022-11-09T02:23:35+00:00"
        },
        {
            "name": "utopia-php/storage",
@ -2368,23 +2434,25 @@
        },
        {
            "name": "utopia-php/system",
-            "version": "0.4.0",
+            "version": "0.6.0",
            "source": {
                "type": "git",
                "url": "https://github.com/utopia-php/system.git",
-                "reference": "67c92c66ce8f0cc925a00bca89f7a188bf9183c0"
+                "reference": "289c4327713deadc9c748b5317d248133a02f245"
            },
            "dist": {
                "type": "zip",
-                "url": "https://api.github.com/repos/utopia-php/system/zipball/67c92c66ce8f0cc925a00bca89f7a188bf9183c0",
-                "reference": "67c92c66ce8f0cc925a00bca89f7a188bf9183c0",
+                "url": "https://api.github.com/repos/utopia-php/system/zipball/289c4327713deadc9c748b5317d248133a02f245",
+                "reference": "289c4327713deadc9c748b5317d248133a02f245",
                "shasum": ""
            },
            "require": {
+                "laravel/pint": "1.2.*",
                "php": ">=7.4"
            },
            "require-dev": {
                "phpunit/phpunit": "^9.3",
+                "squizlabs/php_codesniffer": "^3.6",
                "vimeo/psalm": "4.0.1"
            },
            "type": "library",
@ -2417,9 +2485,9 @@
            ],
            "support": {
                "issues": "https://github.com/utopia-php/system/issues",
-                "source": "https://github.com/utopia-php/system/tree/0.4.0"
+                "source": "https://github.com/utopia-php/system/tree/0.6.0"
            },
-            "time": "2021-02-04T14:14:49+00:00"
+            "time": "2022-11-07T13:51:59+00:00"
        },
        {
            "name": "utopia-php/websocket",
@ -5066,14 +5134,16 @@
    ],
    "aliases": [
        {
-            "package": "utopia-php/database",
-            "version": "0.28.0.0",
-            "alias": "0.26.99",
-            "alias_normalized": "0.26.99.0"
+            "package": "utopia-php/registry",
+            "version": "dev-feat-allow-params",
+            "alias": "0.5.0",
+            "alias_normalized": "0.5.0.0"
        }
    ],
    "minimum-stability": "stable",
-    "stability-flags": [],
+    "stability-flags": {
+        "utopia-php/registry": 20
+    },
    "prefer-stable": false,
    "prefer-lowest": false,
    "platform": {
@ -5097,5 +5167,5 @@
    "platform-overrides": {
        "php": "8.0"
    },
-    "plugin-api-version": "2.2.0"
+    "plugin-api-version": "2.3.0"
 }
--- a/docker-compose.yml
+++ b/docker-compose.yml
@ -14,7 +14,7 @@ version: '3'

 services:
  traefik:
-    image: traefik:2.7
+    image: traefik:2.9
    <<: *x-logging
    container_name: appwrite-traefik
    command:
@ -33,6 +33,17 @@ services:
      - 8080:80
      - 443:443
      - 9500:8080
+    ulimits:
+      nproc: 65535
+      nofile:
+        soft: 65535
+        hard: 65535
+    sysctls:
+      - net.core.somaxconn=1024
+      - net.ipv4.tcp_rmem=1024 4096 16384
+      - net.ipv4.tcp_wmem=1024 4096 16384
+      - net.ipv4.tcp_moderate_rcvbuf=0
+      - net.ipv4.ip_local_port_range=1025 65535
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock
      - appwrite-config:/storage/config:ro
@ -721,7 +732,7 @@ services:
      - MYSQL_DATABASE=${_APP_DB_SCHEMA}
      - MYSQL_USER=${_APP_DB_USER}
      - MYSQL_PASSWORD=${_APP_DB_PASS}
-    command: 'mysqld --innodb-flush-method=fsync --max_connections=49152' # 128 (CPUs) * 6 (Workers) * 64 (Pool size)
+    command: 'mysqld --innodb-flush-method=fsync --max_connections=${_APP_DB_MAX_CONNECTIONS}'
    # command: mv /var/lib/mysql/ib_logfile0 /var/lib/mysql/ib_logfile0.bu && mv /var/lib/mysql/ib_logfile1 /var/lib/mysql/ib_logfile1.bu

  # smtp: