diff --git a/Dockerfile b/Dockerfile index c7316bd1d..698b3fd68 100755 --- a/Dockerfile +++ b/Dockerfile @@ -14,7 +14,7 @@ RUN composer update --ignore-platform-reqs --optimize-autoloader \ FROM php:7.4-cli-alpine as step1 -ENV PHP_REDIS_VERSION=5.3.0 \ +ENV PHP_REDIS_VERSION=5.3.2 \ PHP_SWOOLE_VERSION=v4.5.8 \ PHP_MAXMINDDB_VERSION=v1.8.0 \ PHP_XDEBUG_VERSION=sdebug_2_9-beta @@ -170,6 +170,7 @@ RUN chmod +x /usr/local/bin/doctor && \ chmod +x /usr/local/bin/maintenance && \ chmod +x /usr/local/bin/install && \ chmod +x /usr/local/bin/migrate && \ + chmod +x /usr/local/bin/realtime && \ chmod +x /usr/local/bin/schedule && \ chmod +x /usr/local/bin/sdks && \ chmod +x /usr/local/bin/ssl && \ @@ -195,6 +196,7 @@ RUN echo extension=maxminddb.so >> /usr/local/etc/php/conf.d/maxminddb.ini RUN echo "opcache.preload_user=www-data" >> /usr/local/etc/php/conf.d/appwrite.ini RUN echo "opcache.preload=/usr/src/code/app/preload.php" >> /usr/local/etc/php/conf.d/appwrite.ini RUN echo "opcache.enable_cli=1" >> /usr/local/etc/php/conf.d/appwrite.ini +RUN echo "default_socket_timeout=-1" >> /usr/local/etc/php/conf.d/appwrite.ini EXPOSE 80 diff --git a/app/config/variables.php b/app/config/variables.php index e842283c2..722796c6f 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -105,10 +105,10 @@ return [ ], [ 'name' => '_APP_SYSTEM_SECURITY_EMAIL_ADDRESS', - 'description' => 'This is the email address used to issue SSL certificates for custom domains or the user agent in webhooks. The default value is \'security@localhost.test\'.', + 'description' => 'This is the email address used to issue SSL certificates for custom domains or the user agent in your webhooks payload.', 'introduction' => '0.7.0', - 'default' => 'security@localhost.test', - 'required' => false, + 'default' => '', + 'required' => true, 'question' => '', ], [ @@ -141,6 +141,22 @@ return [ 'required' => false, 'question' => '', ], + [ + 'name' => '_APP_REDIS_USER', + 'description' => 'Redis server user.', + 'introduction' => '0.7', + 'default' => '', + 'required' => false, + 'question' => '', + ], + [ + 'name' => '_APP_REDIS_PASS', + 'description' => 'Redis server password.', + 'introduction' => '0.7', + 'default' => '', + 'required' => false, + 'question' => '', + ], ], ], [ diff --git a/app/controllers/api/health.php b/app/controllers/api/health.php index 696247f3d..c8559278f 100644 --- a/app/controllers/api/health.php +++ b/app/controllers/api/health.php @@ -33,6 +33,20 @@ App::get('/v1/health/version') $response->json(['version' => APP_VERSION_STABLE]); }); +App::get('/v1/health/realtime') + ->desc('Get Realtime') + ->groups(['api', 'health']) + ->label('scope', 'public') + ->inject('response') + ->action(function ($response) { + /** @var Utopia\Response $response */ + $redis = new Redis(); + $redis->connect('redis', 6379); + + $redis->publish('realtime', 'I\'m a live message'); + $response->json(['status' => 'OK']); + }); + App::get('/v1/health/db') ->desc('Get DB') ->groups(['api', 'health']) diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 8dc0c097c..69ffdf4f2 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -110,7 +110,7 @@ App::init(function ($utopia, $request, $response, $project, $user, $register, $e }, ['utopia', 'request', 'response', 'project', 'user', 'register', 'events', 'audits', 'usage', 'deletes'], 'api'); -App::shutdown(function ($utopia, $request, $response, $project, $events, $audits, $usage, $deletes, $mode) { +App::shutdown(function ($utopia, $request, $response, $project, $events, $audits, $usage, $deletes, $realtime, $mode) { /** @var Utopia\App $utopia */ /** @var Utopia\Swoole\Request $request */ /** @var Appwrite\Utopia\Response $response */ @@ -119,6 +119,7 @@ App::shutdown(function ($utopia, $request, $response, $project, $events, $audits /** @var Appwrite\Event\Event $audits */ /** @var Appwrite\Event\Event $usage */ /** @var Appwrite\Event\Event $deletes */ + /** @var Appwrite\Event\Realtime $realtime */ /** @var Appwrite\Event\Event $functions */ /** @var bool $mode */ @@ -139,6 +140,13 @@ App::shutdown(function ($utopia, $request, $response, $project, $events, $audits ->setQueue('v1-functions') ->setClass('FunctionsV1') ->trigger(); + + $realtime + ->setEvent($events->getParam('event')) + ->setProject($project->getId()) + ->setPayload($response->getPayload()) + ->trigger(); + } if (!empty($audits->getParam('event'))) { @@ -162,4 +170,4 @@ App::shutdown(function ($utopia, $request, $response, $project, $events, $audits ; } -}, ['utopia', 'request', 'response', 'project', 'events', 'audits', 'usage', 'deletes', 'mode'], 'api'); +}, ['utopia', 'request', 'response', 'project', 'events', 'audits', 'usage', 'deletes', 'realtime', 'mode'], 'api'); diff --git a/app/controllers/shared/web.php b/app/controllers/shared/web.php index 43eaf74d0..526a4dbb6 100644 --- a/app/controllers/shared/web.php +++ b/app/controllers/shared/web.php @@ -11,13 +11,18 @@ App::init(function ($utopia, $request, $response, $layout) { /* AJAX check */ if (!empty($request->getQuery('version', ''))) { - $layout->setPath(__DIR__.'/../../views/layouts/empty.phtml'); + $layout->setPath(__DIR__ . '/../../views/layouts/empty.phtml'); } - + + $port = $request->getPort(); + $protocol = $request->getProtocol(); + $domain = $request->getHostname(); + $layout ->setParam('title', APP_NAME) - ->setParam('protocol', $request->getProtocol()) - ->setParam('domain', $request->getHostname()) + ->setParam('protocol', $protocol) + ->setParam('domain', $domain) + ->setParam('endpoint', $protocol . '://' . $domain . ($port != 80 && $port != 443 ? ':' . $port : '')) ->setParam('home', App::getEnv('_APP_HOME')) ->setParam('setup', App::getEnv('_APP_SETUP')) ->setParam('class', 'unknown') @@ -34,10 +39,10 @@ App::init(function ($utopia, $request, $response, $layout) { $time = (60 * 60 * 24 * 45); // 45 days cache $response - ->addHeader('Cache-Control', 'public, max-age='.$time) - ->addHeader('Expires', \date('D, d M Y H:i:s', \time() + $time).' GMT') // 45 days cache + ->addHeader('Cache-Control', 'public, max-age=' . $time) + ->addHeader('Expires', \date('D, d M Y H:i:s', \time() + $time) . ' GMT') // 45 days cache ->addHeader('X-Frame-Options', 'SAMEORIGIN') // Avoid console and homepage from showing in iframes - ->addHeader('X-XSS-Protection', '1; mode=block; report=/v1/xss?url='.\urlencode($request->getURI())) + ->addHeader('X-XSS-Protection', '1; mode=block; report=/v1/xss?url=' . \urlencode($request->getURI())) ->addHeader('X-UA-Compatible', 'IE=Edge') // Deny IE browsers from going into quirks mode ; diff --git a/app/init.php b/app/init.php index ed2441d51..98ff4c923 100644 --- a/app/init.php +++ b/app/init.php @@ -20,6 +20,7 @@ use Appwrite\Database\Adapter\Redis as RedisAdapter; use Appwrite\Database\Document; use Appwrite\Database\Validator\Authorization; use Appwrite\Event\Event; +use Appwrite\Event\Realtime; use Appwrite\Extend\PDO; use Appwrite\OpenSSL\OpenSSL; use Utopia\App; @@ -34,7 +35,7 @@ use PDO as PDONative; const APP_NAME = 'Appwrite'; const APP_DOMAIN = 'appwrite.io'; const APP_EMAIL_TEAM = 'team@localhost.test'; // Default email address -const APP_EMAIL_SECURITY = 'security@localhost.test'; // Default security email address +const APP_EMAIL_SECURITY = ''; // Default security email address const APP_USERAGENT = APP_NAME.'-Server v%s. Please report abuse at %s'; const APP_MODE_DEFAULT = 'default'; const APP_MODE_ADMIN = 'admin'; @@ -91,9 +92,13 @@ Config::load('storage-mimes', __DIR__.'/config/storage/mimes.php'); Config::load('storage-inputs', __DIR__.'/config/storage/inputs.php'); Config::load('storage-outputs', __DIR__.'/config/storage/outputs.php'); -Resque::setBackend(App::getEnv('_APP_REDIS_HOST', '') - .':'.App::getEnv('_APP_REDIS_PORT', '')); - +$user = App::getEnv('_APP_REDIS_USER',''); +$pass = App::getEnv('_APP_REDIS_PASS',''); +if(!empty($user) || !empty($pass)) { + Resque::setBackend('redis://'.$user.':'.$pass.'@'.App::getEnv('_APP_REDIS_HOST', '').':'.App::getEnv('_APP_REDIS_PORT', '')); +} else { + Resque::setBackend(App::getEnv('_APP_REDIS_HOST', '').':'.App::getEnv('_APP_REDIS_PORT', '')); +} /** * DB Filters */ @@ -176,6 +181,18 @@ $register->set('statsd', function () { // Register DB connection $register->set('cache', function () { // Register cache connection $redis = new Redis(); $redis->pconnect(App::getEnv('_APP_REDIS_HOST', ''), App::getEnv('_APP_REDIS_PORT', '')); + $user = App::getEnv('_APP_REDIS_USER',''); + $pass = App::getEnv('_APP_REDIS_PASS',''); + $auth = []; + if(!empty($user)) { + $auth["user"] = $user; + } + if(!empty($pass)) { + $auth["pass"] = $pass; + } + if(!empty($auth)) { + $redis->auth($auth); + } $redis->setOption(Redis::OPT_READ_TIMEOUT, -1); return $redis; @@ -305,6 +322,10 @@ App::setResource('events', function($register) { return new Event('', ''); }, ['register']); +App::setResource('realtime', function($register) { + return new Realtime('', '', []); +}, ['register']); + App::setResource('audits', function($register) { return new Event(Event::AUDITS_QUEUE_NAME, Event::AUDITS_CLASS_NAME); }, ['register']); @@ -378,10 +399,10 @@ App::setResource('user', function($mode, $project, $console, $request, $response $request->getCookie(Auth::$cookieName.'_legacy', '')));// Get fallback session from old clients (no SameSite support) // Get fallback session from clients who block 3rd-party cookies - $response->addHeader('X-Debug-Fallback', 'false'); + if($response) $response->addHeader('X-Debug-Fallback', 'false'); if(empty($session['id']) && empty($session['secret'])) { - $response->addHeader('X-Debug-Fallback', 'true'); + if($response) $response->addHeader('X-Debug-Fallback', 'true'); $fallback = $request->getHeader('x-fallback-cookies', ''); $fallback = \json_decode($fallback, true); $session = Auth::decodeSession(((isset($fallback[Auth::$cookieName])) ? $fallback[Auth::$cookieName] : '')); @@ -395,7 +416,7 @@ App::setResource('user', function($mode, $project, $console, $request, $response } else { $user = $consoleDB->getDocument(Auth::$unique); - + $user ->setAttribute('$id', 'admin-'.$user->getAttribute('$id')) ; @@ -410,7 +431,8 @@ App::setResource('user', function($mode, $project, $console, $request, $response if (APP_MODE_ADMIN === $mode) { if (!empty($user->search('teamId', $project->getAttribute('teamId'), $user->getAttribute('memberships')))) { Authorization::setDefaultStatus(false); // Cancel security segmentation for admin users. - } else { + } + else { $user = new Document(['$id' => '', '$collection' => Database::SYSTEM_COLLECTION_USERS]); } } diff --git a/app/realtime.php b/app/realtime.php new file mode 100644 index 000000000..15b32c1bb --- /dev/null +++ b/app/realtime.php @@ -0,0 +1,311 @@ +set([ + 'websocket_compression' => true, + 'package_max_length' => 64000 // Default maximum Package Size (64kb) +]); + +$subscriptions = []; +$connections = []; + +$register->set('redis', function () { + $user = App::getEnv('_APP_REDIS_USER', ''); + $pass = App::getEnv('_APP_REDIS_PASS', ''); + $auth = ''; + if (!empty($user)) { + $auth += $user; + } + if (!empty($pass)) { + $auth += ':' . $pass; + } + + $config = new RedisConfig(); + $config + ->withHost(App::getEnv('_APP_REDIS_HOST', '')) + ->withPort(App::getEnv('_APP_REDIS_PORT', '')) + ->withAuth($auth) + ->withTimeout(0) + ->withReadTimeout(0) + ->withRetryInterval(0); + + + $pool = new RedisPool($config); + + return $pool; +}); + +$server->on('workerStart', function ($server, $workerId) use (&$subscriptions, &$connections, &$register) { + Console::success('Worker ' . ++$workerId . ' started succefully'); + + $attempts = 0; + $start = time(); + + while ($attempts < 300) { + try { + if ($attempts > 0) { + Console::error('Pub/sub connection lost (lasted ' . (time() - $start) . ' seconds, worker: ' . $workerId . '). + Attempting restart in 5 seconds (attempt #' . $attempts . ')'); + sleep(5); // 5 sec delay between connection attempts + } + + $redis = $register->get('redis')->get(); + $redis->setOption(Redis::OPT_READ_TIMEOUT, -1); + + if ($redis->ping(true)) { + $attempts = 0; + Console::success('Pub/sub connection established (worker: ' . $workerId . ')'); + } else { + Console::error('Pub/sub failed (worker: ' . $workerId . ')'); + } + + $redis->subscribe(['realtime'], function ($redis, $channel, $payload) use ($server, &$connections, &$subscriptions) { + /** + * Supported Resources: + * - Collection + * - Document + * - Bucket + * - File + * - User? / Account? (no permissions) + * - Session? (no permissions) + * - Team? (no permissions) + * - Membership? (no permissions) + * - Function + * - Execution + */ + $event = json_decode($payload, true); + + $receivers = Realtime::identifyReceivers($event, $subscriptions); + + foreach ($receivers as $receiver) { + if ($server->exist($receiver) && $server->isEstablished($receiver)) { + $server->push( + $receiver, + json_encode($event['data']), + SWOOLE_WEBSOCKET_OPCODE_TEXT, + SWOOLE_WEBSOCKET_FLAG_FIN | SWOOLE_WEBSOCKET_FLAG_COMPRESS + ); + } else { + $server->close($receiver); + } + } + }); + } catch (\Throwable $th) { + Console::error('Pub/sub error: ' . $th->getMessage()); + $attempts++; + continue; + } + + $attempts++; + } + + Console::error('Failed to restart pub/sub...'); +}); + +$server->on('start', function (Server $server) { + Console::success('Server started succefully'); + + Console::info("Master pid {$server->master_pid}, manager pid {$server->manager_pid}"); + + // listen ctrl + c + Process::signal(2, function () use ($server) { + Console::log('Stop by Ctrl+C'); + $server->shutdown(); + }); +}); + +$server->on('open', function (Server $server, Request $request) use (&$connections, &$subscriptions, &$register) { + Console::info("Connection open (user: {$request->fd}, worker: {$server->getWorkerId()})"); + + $app = new App(''); + $connection = $request->fd; + $request = new SwooleRequest($request); + + App::setResource('request', function () use ($request) { + return $request; + }); + + App::setResource('consoleDB', function () use (&$register) { + $consoleDB = new Database(); + $consoleDB->setAdapter(new RedisAdapter(new MySQLAdapter($register), $register, true)); + $consoleDB->setNamespace('app_console'); // Should be replaced with param if we want to have parent projects + $consoleDB->setMocks(Config::getParam('collections', [])); + + return $consoleDB; + }, ['register']); + + App::setResource('project', function ($consoleDB, $request) { + /** @var Utopia\Swoole\Request $request */ + /** @var Appwrite\Database\Database $consoleDB */ + + Authorization::disable(); + + $project = $consoleDB->getDocument($request->getQuery('project')); + + Authorization::reset(); + + return $project; + }, ['consoleDB', 'request']); + + App::setResource('console', function ($consoleDB) { + return $consoleDB->getDocument('console'); + }, ['consoleDB']); + + App::setResource('user', function ($project, $request, $projectDB) { + /** @var Utopia\Swoole\Request $request */ + /** @var Appwrite\Database\Document $project */ + /** @var Appwrite\Database\Database $projectDB */ + + Authorization::setDefaultStatus(true); + + Auth::setCookieName('a_session_' . $project->getId()); + + $session = Auth::decodeSession( + $request->getCookie( + Auth::$cookieName, // Get sessions + $request->getCookie(Auth::$cookieName . '_legacy', '') + ) + ); // Get fallback session from old clients (no SameSite support) + + Auth::$unique = $session['id']; + Auth::$secret = $session['secret']; + + $user = $projectDB->getDocument(Auth::$unique); + + if ( + empty($user->getId()) // Check a document has been found in the DB + || Database::SYSTEM_COLLECTION_USERS !== $user->getCollection() // Validate returned document is really a user document + || !Auth::tokenVerify($user->getAttribute('tokens', []), Auth::TOKEN_TYPE_LOGIN, Auth::$secret) + ) { // Validate user has valid login token + $user = new Document(['$id' => '', '$collection' => Database::SYSTEM_COLLECTION_USERS]); + } + + return $user; + }, ['project', 'request', 'projectDB']); + + /** @var Appwrite\Database\Document $user */ + $user = $app->getResource('user'); + + /** @var Appwrite\Database\Document $project */ + $project = $app->getResource('project'); + + /** @var Appwrite\Database\Document $console */ + $console = $app->getResource('console'); + + /* + * Project Check + */ + if (empty($project->getId())) { + $server->push($connection, 'Missing or unknown project ID'); + $server->close($connection); + return; + } + + /* + * Abuse Check + */ + $timeLimit = new TimeLimit('url:{url},ip:{ip}', 60, 60, function () use ($register) { + return $register->get('db'); + }); + $timeLimit + ->setNamespace('app_' . $project->getId()) + ->setParam('{ip}', $request->getIP()) + ->setParam('{url}', $request->getURI()); + + $abuse = new Abuse($timeLimit); + + if ($abuse->check() && App::getEnv('_APP_OPTIONS_ABUSE', 'enabled') === 'enabled') { + $server->push($connection, 'Too many requests'); + $server->close($connection); + return; + } + + /* + * Validate Client Domain - Check to avoid CSRF attack + * Adding Appwrite API domains to allow XDOMAIN communication + * Skip this check for non-web platforms which are not required to send an origin header + */ + $origin = $request->getOrigin(); + $originValidator = new Origin(\array_merge($project->getAttribute('platforms', []), $console->getAttribute('platforms', []))); + + if (!$originValidator->isValid($origin)) { + $server->push($connection, $originValidator->getDescription()); + $server->close($connection); + return; + } + + Realtime::setUser($user); + + $roles = Realtime::getRoles(); + $channels = Realtime::parseChannels($request->getQuery('channels', [])); + + /** + * Channels Check + */ + if (empty($channels)) { + $server->push($connection, 'Missing channels'); + $server->close($connection); + return; + } + + Realtime::subscribe($project->getId(), $connection, $roles, $subscriptions, $connections, $channels); + + $server->push($connection, json_encode($channels)); +}); + +$server->on('message', function (Server $server, Frame $frame) { + if ($frame->data === 'reload') { + $server->reload(); + } + + Console::info('Recieved message: ' . $frame->data . ' (user: ' . $frame->fd . ', worker: ' . $server->getWorkerId() . ')'); +}); + +$server->on('close', function (Server $server, int $fd) use (&$connections, &$subscriptions) { + Realtime::unsubscribe($fd, $subscriptions, $connections); + Console::info('Connection close: ' . $fd); +}); + +$server->start(); diff --git a/app/views/console/database/collection.phtml b/app/views/console/database/collection.phtml index afe0365ee..45d3287cf 100644 --- a/app/views/console/database/collection.phtml +++ b/app/views/console/database/collection.phtml @@ -114,7 +114,7 @@ $maxCells = 10; - + {...} diff --git a/app/views/console/database/search/documents.phtml b/app/views/console/database/search/documents.phtml index 1b656dc4e..6fb67feab 100644 --- a/app/views/console/database/search/documents.phtml +++ b/app/views/console/database/search/documents.phtml @@ -72,7 +72,7 @@ $rules = $collection->getAttribute('rules', []); - + {...} diff --git a/app/views/console/database/search/files.phtml b/app/views/console/database/search/files.phtml index f8c39f4a8..ac9f76e0e 100644 --- a/app/views/console/database/search/files.phtml +++ b/app/views/console/database/search/files.phtml @@ -54,7 +54,7 @@ - + diff --git a/app/views/console/functions/function.phtml b/app/views/console/functions/function.phtml index 49e2fffc4..74960eba4 100644 --- a/app/views/console/functions/function.phtml +++ b/app/views/console/functions/function.phtml @@ -596,9 +596,9 @@ $usageStatsEnabled = $this->getParam('usageStatsEnabled',true);

PowerShell

-
diff --git a/app/views/console/settings/index.phtml b/app/views/console/settings/index.phtml index 5eedb6388..45f039d38 100644 --- a/app/views/console/settings/index.phtml +++ b/app/views/console/settings/index.phtml @@ -135,7 +135,7 @@ $customDomainsTarget = $this->getParam('customDomainsTarget', false);
- +