diff --git a/Dockerfile b/Dockerfile
index c7316bd1d..698b3fd68 100755
--- a/Dockerfile
+++ b/Dockerfile
@@ -14,7 +14,7 @@ RUN composer update --ignore-platform-reqs --optimize-autoloader \
FROM php:7.4-cli-alpine as step1
-ENV PHP_REDIS_VERSION=5.3.0 \
+ENV PHP_REDIS_VERSION=5.3.2 \
PHP_SWOOLE_VERSION=v4.5.8 \
PHP_MAXMINDDB_VERSION=v1.8.0 \
PHP_XDEBUG_VERSION=sdebug_2_9-beta
@@ -170,6 +170,7 @@ RUN chmod +x /usr/local/bin/doctor && \
chmod +x /usr/local/bin/maintenance && \
chmod +x /usr/local/bin/install && \
chmod +x /usr/local/bin/migrate && \
+ chmod +x /usr/local/bin/realtime && \
chmod +x /usr/local/bin/schedule && \
chmod +x /usr/local/bin/sdks && \
chmod +x /usr/local/bin/ssl && \
@@ -195,6 +196,7 @@ RUN echo extension=maxminddb.so >> /usr/local/etc/php/conf.d/maxminddb.ini
RUN echo "opcache.preload_user=www-data" >> /usr/local/etc/php/conf.d/appwrite.ini
RUN echo "opcache.preload=/usr/src/code/app/preload.php" >> /usr/local/etc/php/conf.d/appwrite.ini
RUN echo "opcache.enable_cli=1" >> /usr/local/etc/php/conf.d/appwrite.ini
+RUN echo "default_socket_timeout=-1" >> /usr/local/etc/php/conf.d/appwrite.ini
EXPOSE 80
diff --git a/app/config/variables.php b/app/config/variables.php
index e842283c2..722796c6f 100644
--- a/app/config/variables.php
+++ b/app/config/variables.php
@@ -105,10 +105,10 @@ return [
],
[
'name' => '_APP_SYSTEM_SECURITY_EMAIL_ADDRESS',
- 'description' => 'This is the email address used to issue SSL certificates for custom domains or the user agent in webhooks. The default value is \'security@localhost.test\'.',
+ 'description' => 'This is the email address used to issue SSL certificates for custom domains or the user agent in your webhooks payload.',
'introduction' => '0.7.0',
- 'default' => 'security@localhost.test',
- 'required' => false,
+ 'default' => '',
+ 'required' => true,
'question' => '',
],
[
@@ -141,6 +141,22 @@ return [
'required' => false,
'question' => '',
],
+ [
+ 'name' => '_APP_REDIS_USER',
+ 'description' => 'Redis server user.',
+ 'introduction' => '0.7',
+ 'default' => '',
+ 'required' => false,
+ 'question' => '',
+ ],
+ [
+ 'name' => '_APP_REDIS_PASS',
+ 'description' => 'Redis server password.',
+ 'introduction' => '0.7',
+ 'default' => '',
+ 'required' => false,
+ 'question' => '',
+ ],
],
],
[
diff --git a/app/controllers/api/health.php b/app/controllers/api/health.php
index 696247f3d..c8559278f 100644
--- a/app/controllers/api/health.php
+++ b/app/controllers/api/health.php
@@ -33,6 +33,20 @@ App::get('/v1/health/version')
$response->json(['version' => APP_VERSION_STABLE]);
});
+App::get('/v1/health/realtime')
+ ->desc('Get Realtime')
+ ->groups(['api', 'health'])
+ ->label('scope', 'public')
+ ->inject('response')
+ ->action(function ($response) {
+ /** @var Utopia\Response $response */
+ $redis = new Redis();
+ $redis->connect('redis', 6379);
+
+ $redis->publish('realtime', 'I\'m a live message');
+ $response->json(['status' => 'OK']);
+ });
+
App::get('/v1/health/db')
->desc('Get DB')
->groups(['api', 'health'])
diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php
index 8dc0c097c..69ffdf4f2 100644
--- a/app/controllers/shared/api.php
+++ b/app/controllers/shared/api.php
@@ -110,7 +110,7 @@ App::init(function ($utopia, $request, $response, $project, $user, $register, $e
}, ['utopia', 'request', 'response', 'project', 'user', 'register', 'events', 'audits', 'usage', 'deletes'], 'api');
-App::shutdown(function ($utopia, $request, $response, $project, $events, $audits, $usage, $deletes, $mode) {
+App::shutdown(function ($utopia, $request, $response, $project, $events, $audits, $usage, $deletes, $realtime, $mode) {
/** @var Utopia\App $utopia */
/** @var Utopia\Swoole\Request $request */
/** @var Appwrite\Utopia\Response $response */
@@ -119,6 +119,7 @@ App::shutdown(function ($utopia, $request, $response, $project, $events, $audits
/** @var Appwrite\Event\Event $audits */
/** @var Appwrite\Event\Event $usage */
/** @var Appwrite\Event\Event $deletes */
+ /** @var Appwrite\Event\Realtime $realtime */
/** @var Appwrite\Event\Event $functions */
/** @var bool $mode */
@@ -139,6 +140,13 @@ App::shutdown(function ($utopia, $request, $response, $project, $events, $audits
->setQueue('v1-functions')
->setClass('FunctionsV1')
->trigger();
+
+ $realtime
+ ->setEvent($events->getParam('event'))
+ ->setProject($project->getId())
+ ->setPayload($response->getPayload())
+ ->trigger();
+
}
if (!empty($audits->getParam('event'))) {
@@ -162,4 +170,4 @@ App::shutdown(function ($utopia, $request, $response, $project, $events, $audits
;
}
-}, ['utopia', 'request', 'response', 'project', 'events', 'audits', 'usage', 'deletes', 'mode'], 'api');
+}, ['utopia', 'request', 'response', 'project', 'events', 'audits', 'usage', 'deletes', 'realtime', 'mode'], 'api');
diff --git a/app/controllers/shared/web.php b/app/controllers/shared/web.php
index 43eaf74d0..526a4dbb6 100644
--- a/app/controllers/shared/web.php
+++ b/app/controllers/shared/web.php
@@ -11,13 +11,18 @@ App::init(function ($utopia, $request, $response, $layout) {
/* AJAX check */
if (!empty($request->getQuery('version', ''))) {
- $layout->setPath(__DIR__.'/../../views/layouts/empty.phtml');
+ $layout->setPath(__DIR__ . '/../../views/layouts/empty.phtml');
}
-
+
+ $port = $request->getPort();
+ $protocol = $request->getProtocol();
+ $domain = $request->getHostname();
+
$layout
->setParam('title', APP_NAME)
- ->setParam('protocol', $request->getProtocol())
- ->setParam('domain', $request->getHostname())
+ ->setParam('protocol', $protocol)
+ ->setParam('domain', $domain)
+ ->setParam('endpoint', $protocol . '://' . $domain . ($port != 80 && $port != 443 ? ':' . $port : ''))
->setParam('home', App::getEnv('_APP_HOME'))
->setParam('setup', App::getEnv('_APP_SETUP'))
->setParam('class', 'unknown')
@@ -34,10 +39,10 @@ App::init(function ($utopia, $request, $response, $layout) {
$time = (60 * 60 * 24 * 45); // 45 days cache
$response
- ->addHeader('Cache-Control', 'public, max-age='.$time)
- ->addHeader('Expires', \date('D, d M Y H:i:s', \time() + $time).' GMT') // 45 days cache
+ ->addHeader('Cache-Control', 'public, max-age=' . $time)
+ ->addHeader('Expires', \date('D, d M Y H:i:s', \time() + $time) . ' GMT') // 45 days cache
->addHeader('X-Frame-Options', 'SAMEORIGIN') // Avoid console and homepage from showing in iframes
- ->addHeader('X-XSS-Protection', '1; mode=block; report=/v1/xss?url='.\urlencode($request->getURI()))
+ ->addHeader('X-XSS-Protection', '1; mode=block; report=/v1/xss?url=' . \urlencode($request->getURI()))
->addHeader('X-UA-Compatible', 'IE=Edge') // Deny IE browsers from going into quirks mode
;
diff --git a/app/init.php b/app/init.php
index ed2441d51..98ff4c923 100644
--- a/app/init.php
+++ b/app/init.php
@@ -20,6 +20,7 @@ use Appwrite\Database\Adapter\Redis as RedisAdapter;
use Appwrite\Database\Document;
use Appwrite\Database\Validator\Authorization;
use Appwrite\Event\Event;
+use Appwrite\Event\Realtime;
use Appwrite\Extend\PDO;
use Appwrite\OpenSSL\OpenSSL;
use Utopia\App;
@@ -34,7 +35,7 @@ use PDO as PDONative;
const APP_NAME = 'Appwrite';
const APP_DOMAIN = 'appwrite.io';
const APP_EMAIL_TEAM = 'team@localhost.test'; // Default email address
-const APP_EMAIL_SECURITY = 'security@localhost.test'; // Default security email address
+const APP_EMAIL_SECURITY = ''; // Default security email address
const APP_USERAGENT = APP_NAME.'-Server v%s. Please report abuse at %s';
const APP_MODE_DEFAULT = 'default';
const APP_MODE_ADMIN = 'admin';
@@ -91,9 +92,13 @@ Config::load('storage-mimes', __DIR__.'/config/storage/mimes.php');
Config::load('storage-inputs', __DIR__.'/config/storage/inputs.php');
Config::load('storage-outputs', __DIR__.'/config/storage/outputs.php');
-Resque::setBackend(App::getEnv('_APP_REDIS_HOST', '')
- .':'.App::getEnv('_APP_REDIS_PORT', ''));
-
+$user = App::getEnv('_APP_REDIS_USER','');
+$pass = App::getEnv('_APP_REDIS_PASS','');
+if(!empty($user) || !empty($pass)) {
+ Resque::setBackend('redis://'.$user.':'.$pass.'@'.App::getEnv('_APP_REDIS_HOST', '').':'.App::getEnv('_APP_REDIS_PORT', ''));
+} else {
+ Resque::setBackend(App::getEnv('_APP_REDIS_HOST', '').':'.App::getEnv('_APP_REDIS_PORT', ''));
+}
/**
* DB Filters
*/
@@ -176,6 +181,18 @@ $register->set('statsd', function () { // Register DB connection
$register->set('cache', function () { // Register cache connection
$redis = new Redis();
$redis->pconnect(App::getEnv('_APP_REDIS_HOST', ''), App::getEnv('_APP_REDIS_PORT', ''));
+ $user = App::getEnv('_APP_REDIS_USER','');
+ $pass = App::getEnv('_APP_REDIS_PASS','');
+ $auth = [];
+ if(!empty($user)) {
+ $auth["user"] = $user;
+ }
+ if(!empty($pass)) {
+ $auth["pass"] = $pass;
+ }
+ if(!empty($auth)) {
+ $redis->auth($auth);
+ }
$redis->setOption(Redis::OPT_READ_TIMEOUT, -1);
return $redis;
@@ -305,6 +322,10 @@ App::setResource('events', function($register) {
return new Event('', '');
}, ['register']);
+App::setResource('realtime', function($register) {
+ return new Realtime('', '', []);
+}, ['register']);
+
App::setResource('audits', function($register) {
return new Event(Event::AUDITS_QUEUE_NAME, Event::AUDITS_CLASS_NAME);
}, ['register']);
@@ -378,10 +399,10 @@ App::setResource('user', function($mode, $project, $console, $request, $response
$request->getCookie(Auth::$cookieName.'_legacy', '')));// Get fallback session from old clients (no SameSite support)
// Get fallback session from clients who block 3rd-party cookies
- $response->addHeader('X-Debug-Fallback', 'false');
+ if($response) $response->addHeader('X-Debug-Fallback', 'false');
if(empty($session['id']) && empty($session['secret'])) {
- $response->addHeader('X-Debug-Fallback', 'true');
+ if($response) $response->addHeader('X-Debug-Fallback', 'true');
$fallback = $request->getHeader('x-fallback-cookies', '');
$fallback = \json_decode($fallback, true);
$session = Auth::decodeSession(((isset($fallback[Auth::$cookieName])) ? $fallback[Auth::$cookieName] : ''));
@@ -395,7 +416,7 @@ App::setResource('user', function($mode, $project, $console, $request, $response
}
else {
$user = $consoleDB->getDocument(Auth::$unique);
-
+
$user
->setAttribute('$id', 'admin-'.$user->getAttribute('$id'))
;
@@ -410,7 +431,8 @@ App::setResource('user', function($mode, $project, $console, $request, $response
if (APP_MODE_ADMIN === $mode) {
if (!empty($user->search('teamId', $project->getAttribute('teamId'), $user->getAttribute('memberships')))) {
Authorization::setDefaultStatus(false); // Cancel security segmentation for admin users.
- } else {
+ }
+ else {
$user = new Document(['$id' => '', '$collection' => Database::SYSTEM_COLLECTION_USERS]);
}
}
diff --git a/app/realtime.php b/app/realtime.php
new file mode 100644
index 000000000..15b32c1bb
--- /dev/null
+++ b/app/realtime.php
@@ -0,0 +1,311 @@
+set([
+ 'websocket_compression' => true,
+ 'package_max_length' => 64000 // Default maximum Package Size (64kb)
+]);
+
+$subscriptions = [];
+$connections = [];
+
+$register->set('redis', function () {
+ $user = App::getEnv('_APP_REDIS_USER', '');
+ $pass = App::getEnv('_APP_REDIS_PASS', '');
+ $auth = '';
+ if (!empty($user)) {
+ $auth += $user;
+ }
+ if (!empty($pass)) {
+ $auth += ':' . $pass;
+ }
+
+ $config = new RedisConfig();
+ $config
+ ->withHost(App::getEnv('_APP_REDIS_HOST', ''))
+ ->withPort(App::getEnv('_APP_REDIS_PORT', ''))
+ ->withAuth($auth)
+ ->withTimeout(0)
+ ->withReadTimeout(0)
+ ->withRetryInterval(0);
+
+
+ $pool = new RedisPool($config);
+
+ return $pool;
+});
+
+$server->on('workerStart', function ($server, $workerId) use (&$subscriptions, &$connections, &$register) {
+ Console::success('Worker ' . ++$workerId . ' started succefully');
+
+ $attempts = 0;
+ $start = time();
+
+ while ($attempts < 300) {
+ try {
+ if ($attempts > 0) {
+ Console::error('Pub/sub connection lost (lasted ' . (time() - $start) . ' seconds, worker: ' . $workerId . ').
+ Attempting restart in 5 seconds (attempt #' . $attempts . ')');
+ sleep(5); // 5 sec delay between connection attempts
+ }
+
+ $redis = $register->get('redis')->get();
+ $redis->setOption(Redis::OPT_READ_TIMEOUT, -1);
+
+ if ($redis->ping(true)) {
+ $attempts = 0;
+ Console::success('Pub/sub connection established (worker: ' . $workerId . ')');
+ } else {
+ Console::error('Pub/sub failed (worker: ' . $workerId . ')');
+ }
+
+ $redis->subscribe(['realtime'], function ($redis, $channel, $payload) use ($server, &$connections, &$subscriptions) {
+ /**
+ * Supported Resources:
+ * - Collection
+ * - Document
+ * - Bucket
+ * - File
+ * - User? / Account? (no permissions)
+ * - Session? (no permissions)
+ * - Team? (no permissions)
+ * - Membership? (no permissions)
+ * - Function
+ * - Execution
+ */
+ $event = json_decode($payload, true);
+
+ $receivers = Realtime::identifyReceivers($event, $subscriptions);
+
+ foreach ($receivers as $receiver) {
+ if ($server->exist($receiver) && $server->isEstablished($receiver)) {
+ $server->push(
+ $receiver,
+ json_encode($event['data']),
+ SWOOLE_WEBSOCKET_OPCODE_TEXT,
+ SWOOLE_WEBSOCKET_FLAG_FIN | SWOOLE_WEBSOCKET_FLAG_COMPRESS
+ );
+ } else {
+ $server->close($receiver);
+ }
+ }
+ });
+ } catch (\Throwable $th) {
+ Console::error('Pub/sub error: ' . $th->getMessage());
+ $attempts++;
+ continue;
+ }
+
+ $attempts++;
+ }
+
+ Console::error('Failed to restart pub/sub...');
+});
+
+$server->on('start', function (Server $server) {
+ Console::success('Server started succefully');
+
+ Console::info("Master pid {$server->master_pid}, manager pid {$server->manager_pid}");
+
+ // listen ctrl + c
+ Process::signal(2, function () use ($server) {
+ Console::log('Stop by Ctrl+C');
+ $server->shutdown();
+ });
+});
+
+$server->on('open', function (Server $server, Request $request) use (&$connections, &$subscriptions, &$register) {
+ Console::info("Connection open (user: {$request->fd}, worker: {$server->getWorkerId()})");
+
+ $app = new App('');
+ $connection = $request->fd;
+ $request = new SwooleRequest($request);
+
+ App::setResource('request', function () use ($request) {
+ return $request;
+ });
+
+ App::setResource('consoleDB', function () use (&$register) {
+ $consoleDB = new Database();
+ $consoleDB->setAdapter(new RedisAdapter(new MySQLAdapter($register), $register, true));
+ $consoleDB->setNamespace('app_console'); // Should be replaced with param if we want to have parent projects
+ $consoleDB->setMocks(Config::getParam('collections', []));
+
+ return $consoleDB;
+ }, ['register']);
+
+ App::setResource('project', function ($consoleDB, $request) {
+ /** @var Utopia\Swoole\Request $request */
+ /** @var Appwrite\Database\Database $consoleDB */
+
+ Authorization::disable();
+
+ $project = $consoleDB->getDocument($request->getQuery('project'));
+
+ Authorization::reset();
+
+ return $project;
+ }, ['consoleDB', 'request']);
+
+ App::setResource('console', function ($consoleDB) {
+ return $consoleDB->getDocument('console');
+ }, ['consoleDB']);
+
+ App::setResource('user', function ($project, $request, $projectDB) {
+ /** @var Utopia\Swoole\Request $request */
+ /** @var Appwrite\Database\Document $project */
+ /** @var Appwrite\Database\Database $projectDB */
+
+ Authorization::setDefaultStatus(true);
+
+ Auth::setCookieName('a_session_' . $project->getId());
+
+ $session = Auth::decodeSession(
+ $request->getCookie(
+ Auth::$cookieName, // Get sessions
+ $request->getCookie(Auth::$cookieName . '_legacy', '')
+ )
+ ); // Get fallback session from old clients (no SameSite support)
+
+ Auth::$unique = $session['id'];
+ Auth::$secret = $session['secret'];
+
+ $user = $projectDB->getDocument(Auth::$unique);
+
+ if (
+ empty($user->getId()) // Check a document has been found in the DB
+ || Database::SYSTEM_COLLECTION_USERS !== $user->getCollection() // Validate returned document is really a user document
+ || !Auth::tokenVerify($user->getAttribute('tokens', []), Auth::TOKEN_TYPE_LOGIN, Auth::$secret)
+ ) { // Validate user has valid login token
+ $user = new Document(['$id' => '', '$collection' => Database::SYSTEM_COLLECTION_USERS]);
+ }
+
+ return $user;
+ }, ['project', 'request', 'projectDB']);
+
+ /** @var Appwrite\Database\Document $user */
+ $user = $app->getResource('user');
+
+ /** @var Appwrite\Database\Document $project */
+ $project = $app->getResource('project');
+
+ /** @var Appwrite\Database\Document $console */
+ $console = $app->getResource('console');
+
+ /*
+ * Project Check
+ */
+ if (empty($project->getId())) {
+ $server->push($connection, 'Missing or unknown project ID');
+ $server->close($connection);
+ return;
+ }
+
+ /*
+ * Abuse Check
+ */
+ $timeLimit = new TimeLimit('url:{url},ip:{ip}', 60, 60, function () use ($register) {
+ return $register->get('db');
+ });
+ $timeLimit
+ ->setNamespace('app_' . $project->getId())
+ ->setParam('{ip}', $request->getIP())
+ ->setParam('{url}', $request->getURI());
+
+ $abuse = new Abuse($timeLimit);
+
+ if ($abuse->check() && App::getEnv('_APP_OPTIONS_ABUSE', 'enabled') === 'enabled') {
+ $server->push($connection, 'Too many requests');
+ $server->close($connection);
+ return;
+ }
+
+ /*
+ * Validate Client Domain - Check to avoid CSRF attack
+ * Adding Appwrite API domains to allow XDOMAIN communication
+ * Skip this check for non-web platforms which are not required to send an origin header
+ */
+ $origin = $request->getOrigin();
+ $originValidator = new Origin(\array_merge($project->getAttribute('platforms', []), $console->getAttribute('platforms', [])));
+
+ if (!$originValidator->isValid($origin)) {
+ $server->push($connection, $originValidator->getDescription());
+ $server->close($connection);
+ return;
+ }
+
+ Realtime::setUser($user);
+
+ $roles = Realtime::getRoles();
+ $channels = Realtime::parseChannels($request->getQuery('channels', []));
+
+ /**
+ * Channels Check
+ */
+ if (empty($channels)) {
+ $server->push($connection, 'Missing channels');
+ $server->close($connection);
+ return;
+ }
+
+ Realtime::subscribe($project->getId(), $connection, $roles, $subscriptions, $connections, $channels);
+
+ $server->push($connection, json_encode($channels));
+});
+
+$server->on('message', function (Server $server, Frame $frame) {
+ if ($frame->data === 'reload') {
+ $server->reload();
+ }
+
+ Console::info('Recieved message: ' . $frame->data . ' (user: ' . $frame->fd . ', worker: ' . $server->getWorkerId() . ')');
+});
+
+$server->on('close', function (Server $server, int $fd) use (&$connections, &$subscriptions) {
+ Realtime::unsubscribe($fd, $subscriptions, $connections);
+ Console::info('Connection close: ' . $fd);
+});
+
+$server->start();
diff --git a/app/views/console/database/collection.phtml b/app/views/console/database/collection.phtml
index afe0365ee..45d3287cf 100644
--- a/app/views/console/database/collection.phtml
+++ b/app/views/console/database/collection.phtml
@@ -114,7 +114,7 @@ $maxCells = 10;
-
+
{...}
diff --git a/app/views/console/database/search/documents.phtml b/app/views/console/database/search/documents.phtml
index 1b656dc4e..6fb67feab 100644
--- a/app/views/console/database/search/documents.phtml
+++ b/app/views/console/database/search/documents.phtml
@@ -72,7 +72,7 @@ $rules = $collection->getAttribute('rules', []);
-
+
{...}
diff --git a/app/views/console/database/search/files.phtml b/app/views/console/database/search/files.phtml
index f8c39f4a8..ac9f76e0e 100644
--- a/app/views/console/database/search/files.phtml
+++ b/app/views/console/database/search/files.phtml
@@ -54,7 +54,7 @@
PowerShell
To complete set up, add this OAuth2 redirect URI to your escape(ucfirst($provider)); ?> app configuration.