From c6f9f2fc1e22b0a8f076c240a23804c1f0266c75 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Wed, 13 Jan 2021 17:51:02 +0100 Subject: [PATCH 001/156] feat(migration): v06 - first draft --- app/tasks/migrate.php | 200 +++---------------------- src/Appwrite/Migration/Migration.php | 41 +++++ src/Appwrite/Migration/Version/V04.php | 14 ++ src/Appwrite/Migration/Version/V05.php | 164 ++++++++++++++++++++ src/Appwrite/Migration/Version/V06.php | 17 +++ 5 files changed, 256 insertions(+), 180 deletions(-) create mode 100644 src/Appwrite/Migration/Migration.php create mode 100644 src/Appwrite/Migration/Version/V04.php create mode 100644 src/Appwrite/Migration/Version/V05.php create mode 100644 src/Appwrite/Migration/Version/V06.php diff --git a/app/tasks/migrate.php b/app/tasks/migrate.php index bba8ad416..c25763dde 100644 --- a/app/tasks/migrate.php +++ b/app/tasks/migrate.php @@ -5,187 +5,26 @@ global $cli, $register, $projectDB, $console; use Utopia\Config\Config; use Utopia\CLI\Console; use Appwrite\Database\Database; -use Appwrite\Database\Document; use Appwrite\Database\Validator\Authorization; use Appwrite\Database\Adapter\MySQL as MySQLAdapter; use Appwrite\Database\Adapter\Redis as RedisAdapter; - -$callbacks = [ - '0.4.0' => function() { - Console::log('I got nothing to do.'); - }, - - '0.5.0' => function($project) use ($register, $projectDB) { - $db = $register->get('db'); - - Console::log('Migrating project: '.$project->getAttribute('name').' ('.$project->getId().')'); - - // Update all documents $uid -> $id - - $limit = 30; - $sum = 30; - $offset = 0; - - while ($sum >= 30) { - $all = $projectDB->getCollection([ - 'limit' => $limit, - 'offset' => $offset, - 'orderType' => 'DESC', - ]); - - $sum = \count($all); - - Console::log('Migrating: '.$offset.' / '.$projectDB->getSum()); - - foreach($all as $document) { - $document = fixDocument($document); - - if(empty($document->getId())) { - throw new Exception('Missing ID'); - } - - try { - $new = $projectDB->overwriteDocument($document->getArrayCopy()); - } catch (\Throwable $th) { - var_dump($document); - Console::error('Failed to update document: '.$th->getMessage()); - continue; - } - - if($new->getId() !== $document->getId()) { - throw new Exception('Duplication Error'); - } - } - - $offset = $offset + $limit; - } - - $schema = $_SERVER['_APP_DB_SCHEMA'] ?? ''; - - try { - $statement = $db->prepare(" - - CREATE TABLE IF NOT EXISTS `template.database.unique` ( - `id` int(10) unsigned NOT NULL AUTO_INCREMENT, - `key` varchar(128) DEFAULT NULL, - PRIMARY KEY (`id`), - UNIQUE KEY `index1` (`key`) - ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - - CREATE TABLE IF NOT EXISTS `{$schema}`.`app_{$project->getId()}.database.unique` LIKE `template.database.unique`; - ALTER TABLE `{$schema}`.`app_{$project->getId()}.audit.audit` DROP COLUMN IF EXISTS `userType`; - ALTER TABLE `{$schema}`.`app_{$project->getId()}.audit.audit` DROP INDEX IF EXISTS `index_1`; - ALTER TABLE `{$schema}`.`app_{$project->getId()}.audit.audit` ADD INDEX IF NOT EXISTS `index_1` (`userId` ASC); - "); - - $statement->closeCursor(); - - $statement->execute(); - } - catch (\Exception $e) { - Console::error('Failed to alter table for project: '.$project->getId().' with message: '.$e->getMessage().'/'); - } - }, -]; - -function fixDocument(Document $document) { - $providers = Config::getParam('providers'); - - if($document->getAttribute('$collection') === Database::SYSTEM_COLLECTION_PROJECTS){ - foreach($providers as $key => $provider) { - if(!empty($document->getAttribute('usersOauth'.\ucfirst($key).'Appid'))) { - $document - ->setAttribute('usersOauth2'.\ucfirst($key).'Appid', $document->getAttribute('usersOauth'.\ucfirst($key).'Appid', '')) - ->removeAttribute('usersOauth'.\ucfirst($key).'Appid') - ; - } - - if(!empty($document->getAttribute('usersOauth'.\ucfirst($key).'Secret'))) { - $document - ->setAttribute('usersOauth2'.\ucfirst($key).'Secret', $document->getAttribute('usersOauth'.\ucfirst($key).'Secret', '')) - ->removeAttribute('usersOauth'.\ucfirst($key).'Secret') - ; - } - } - } - - if($document->getAttribute('$collection') === Database::SYSTEM_COLLECTION_WEBHOOKS){ - $document->setAttribute('security', ($document->getAttribute('security')) ? true : false); - } - - if($document->getAttribute('$collection') === Database::SYSTEM_COLLECTION_TASKS){ - $document->setAttribute('security', ($document->getAttribute('security')) ? true : false); - } - - if($document->getAttribute('$collection') === Database::SYSTEM_COLLECTION_USERS) { - foreach($providers as $key => $provider) { - if(!empty($document->getAttribute('oauth'.\ucfirst($key)))) { - $document - ->setAttribute('oauth2'.\ucfirst($key), $document->getAttribute('oauth'.\ucfirst($key), '')) - ->removeAttribute('oauth'.\ucfirst($key)) - ; - } - - if(!empty($document->getAttribute('oauth'.\ucfirst($key).'AccessToken'))) { - $document - ->setAttribute('oauth2'.\ucfirst($key).'AccessToken', $document->getAttribute('oauth'.\ucfirst($key).'AccessToken', '')) - ->removeAttribute('oauth'.\ucfirst($key).'AccessToken') - ; - } - } - - if($document->getAttribute('confirm', null) !== null) { - $document - ->setAttribute('emailVerification', $document->getAttribute('confirm', $document->getAttribute('emailVerification', false))) - ->removeAttribute('confirm') - ; - } - } - - if($document->getAttribute('$collection') === Database::SYSTEM_COLLECTION_PLATFORMS) { - if($document->getAttribute('url', null) !== null) { - $document - ->setAttribute('hostname', \parse_url($document->getAttribute('url', $document->getAttribute('hostname', '')), PHP_URL_HOST)) - ->removeAttribute('url') - ; - } - } - - $document - ->setAttribute('$id', $document->getAttribute('$uid', $document->getAttribute('$id'))) - ->removeAttribute('$uid') - ; - - foreach($document as &$attr) { // Handle child documents - if($attr instanceof Document) { - $attr = fixDocument($attr); - } - - if(\is_array($attr)) { - foreach($attr as &$child) { - if($child instanceof Document) { - $child = fixDocument($child); - } - } - } - } - - return $document; -} +use Appwrite\Migration\Version; $cli ->task('migrate') - ->action(function () use ($register, $callbacks) { + ->action(function () use ($register) { Console::success('Starting Data Migration'); $consoleDB = new Database(); - $consoleDB->setAdapter(new RedisAdapter(new MySQLAdapter($register), $register)); - $consoleDB->setNamespace('app_console'); // Main DB - $consoleDB->setMocks(Config::getParam('collections', [])); - + $consoleDB + ->setAdapter(new RedisAdapter(new MySQLAdapter($register), $register)) + ->setNamespace('app_console') // Main DB + ->setMocks(Config::getParam('collections', [])); + $projectDB = new Database(); - $projectDB->setAdapter(new RedisAdapter(new MySQLAdapter($register), $register)); - $projectDB->setMocks(Config::getParam('collections', [])); + $projectDB + ->setAdapter(new RedisAdapter(new MySQLAdapter($register), $register)) + ->setMocks(Config::getParam('collections', [])); $console = $consoleDB->getDocument('console'); @@ -197,16 +36,17 @@ $cli $projects = [$console]; $count = 0; - while ($sum >= 30) { - foreach($projects as $project) { - - $projectDB->setNamespace('app_'.$project->getId()); + $migration = new Version\V06($register->get('db')); //TODO: remove hardcoded version and move to dynamic migration + while ($sum >= 30) { + foreach ($projects as $project) { try { - $callbacks['0.5.0']($project, $projectDB); + $migration + ->setProject($project, $projectDB) + ->execute(); } catch (\Throwable $th) { throw $th; - Console::error('Failed to update project ("'.$project->getId().'") version with error: '.$th->getMessage()); + Console::error('Failed to update project ("' . $project->getId() . '") version with error: ' . $th->getMessage()); } } @@ -214,7 +54,7 @@ $cli 'limit' => $limit, 'offset' => $offset, 'filters' => [ - '$collection='.Database::SYSTEM_COLLECTION_PROJECTS, + '$collection=' . Database::SYSTEM_COLLECTION_PROJECTS, ], ]); @@ -222,8 +62,8 @@ $cli $offset = $offset + $limit; $count = $count + $sum; - Console::log('Fetched '.$count.'/'.$consoleDB->getSum().' projects...'); + Console::log('Fetched ' . $count . '/' . $consoleDB->getSum() . ' projects...'); } Console::success('Data Migration Completed'); - }); \ No newline at end of file + }); diff --git a/src/Appwrite/Migration/Migration.php b/src/Appwrite/Migration/Migration.php new file mode 100644 index 000000000..f2d6a8c0a --- /dev/null +++ b/src/Appwrite/Migration/Migration.php @@ -0,0 +1,41 @@ +db = $db; + } + + /** + * Set project for migration. + */ + public function setProject(Document $project, Database $projectDB) + { + $this->project = $project; + $this->projectDB = $projectDB; + $this->projectDB->setNamespace('app_'.$project->getId()); + return $this; + } + + /** + * Executes migration for set project. + */ + abstract public function execute(): void; +} diff --git a/src/Appwrite/Migration/Version/V04.php b/src/Appwrite/Migration/Version/V04.php new file mode 100644 index 000000000..f1f879d89 --- /dev/null +++ b/src/Appwrite/Migration/Version/V04.php @@ -0,0 +1,14 @@ +db; + $project = $this->project; + $projectDB = $this->projectDB; + Console::log('Migrating project: ' . $project->getAttribute('name') . ' (' . $project->getId() . ')'); + + // Update all documents $uid -> $id + + $limit = 30; + $sum = 30; + $offset = 0; + + while ($sum >= 30) { + $all = $projectDB->getCollection([ + 'limit' => $limit, + 'offset' => $offset, + 'orderType' => 'DESC', + ]); + + $sum = \count($all); + + Console::log('Migrating: ' . $offset . ' / ' . $projectDB->getSum()); + + foreach ($all as $document) { + $document = $this->fixDocument($document); + + if (empty($document->getId())) { + throw new Exception('Missing ID'); + } + + try { + $new = $projectDB->overwriteDocument($document->getArrayCopy()); + } catch (\Throwable $th) { + var_dump($document); + Console::error('Failed to update document: ' . $th->getMessage()); + continue; + } + + if ($new->getId() !== $document->getId()) { + throw new Exception('Duplication Error'); + } + } + + $offset = $offset + $limit; + } + + $schema = $_SERVER['_APP_DB_SCHEMA'] ?? ''; + + try { + $statement = $db->prepare(" + + CREATE TABLE IF NOT EXISTS `template.database.unique` ( + `id` int(10) unsigned NOT NULL AUTO_INCREMENT, + `key` varchar(128) DEFAULT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `index1` (`key`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + + CREATE TABLE IF NOT EXISTS `{$schema}`.`app_{$project->getId()}.database.unique` LIKE `template.database.unique`; + ALTER TABLE `{$schema}`.`app_{$project->getId()}.audit.audit` DROP COLUMN IF EXISTS `userType`; + ALTER TABLE `{$schema}`.`app_{$project->getId()}.audit.audit` DROP INDEX IF EXISTS `index_1`; + ALTER TABLE `{$schema}`.`app_{$project->getId()}.audit.audit` ADD INDEX IF NOT EXISTS `index_1` (`userId` ASC); + "); + + $statement->closeCursor(); + + $statement->execute(); + } catch (\Exception $e) { + Console::error('Failed to alter table for project: ' . $project->getId() . ' with message: ' . $e->getMessage() . '/'); + } + } + + private function fixDocument(Document $document) + { + $providers = Config::getParam('providers'); + + switch ($document->getAttribute('$collection')) { + case Database::SYSTEM_COLLECTION_PROJECTS: + foreach ($providers as $key => $provider) { + if (!empty($document->getAttribute('usersOauth' . \ucfirst($key) . 'Appid'))) { + $document + ->setAttribute('usersOauth2' . \ucfirst($key) . 'Appid', $document->getAttribute('usersOauth' . \ucfirst($key) . 'Appid', '')) + ->removeAttribute('usersOauth' . \ucfirst($key) . 'Appid'); + } + + if (!empty($document->getAttribute('usersOauth' . \ucfirst($key) . 'Secret'))) { + $document + ->setAttribute('usersOauth2' . \ucfirst($key) . 'Secret', $document->getAttribute('usersOauth' . \ucfirst($key) . 'Secret', '')) + ->removeAttribute('usersOauth' . \ucfirst($key) . 'Secret'); + } + } + break; + + case Database::SYSTEM_COLLECTION_PROJECTS: + case Database::SYSTEM_COLLECTION_TASKS: + $document->setAttribute('security', ($document->getAttribute('security')) ? true : false); + break; + + case Database::SYSTEM_COLLECTION_USERS: + foreach ($providers as $key => $provider) { + if (!empty($document->getAttribute('oauth' . \ucfirst($key)))) { + $document + ->setAttribute('oauth2' . \ucfirst($key), $document->getAttribute('oauth' . \ucfirst($key), '')) + ->removeAttribute('oauth' . \ucfirst($key)); + } + + if (!empty($document->getAttribute('oauth' . \ucfirst($key) . 'AccessToken'))) { + $document + ->setAttribute('oauth2' . \ucfirst($key) . 'AccessToken', $document->getAttribute('oauth' . \ucfirst($key) . 'AccessToken', '')) + ->removeAttribute('oauth' . \ucfirst($key) . 'AccessToken'); + } + } + + if ($document->getAttribute('confirm', null) !== null) { + $document + ->setAttribute('emailVerification', $document->getAttribute('confirm', $document->getAttribute('emailVerification', false))) + ->removeAttribute('confirm'); + } + break; + + case Database::SYSTEM_COLLECTION_PLATFORMS: + if ($document->getAttribute('url', null) !== null) { + $document + ->setAttribute('hostname', \parse_url($document->getAttribute('url', $document->getAttribute('hostname', '')), PHP_URL_HOST)) + ->removeAttribute('url'); + } + break; + } + + $document + ->setAttribute('$id', $document->getAttribute('$uid', $document->getAttribute('$id'))) + ->removeAttribute('$uid'); + + foreach ($document as &$attr) { // Handle child documents + if ($attr instanceof Document) { + $attr = $this->fixDocument($attr); + } + + if (\is_array($attr)) { + foreach ($attr as &$child) { + if ($child instanceof Document) { + $child = $this->fixDocument($child); + } + } + } + } + + return $document; + } +} diff --git a/src/Appwrite/Migration/Version/V06.php b/src/Appwrite/Migration/Version/V06.php new file mode 100644 index 000000000..33ddafe81 --- /dev/null +++ b/src/Appwrite/Migration/Version/V06.php @@ -0,0 +1,17 @@ + Date: Wed, 13 Jan 2021 17:58:01 +0100 Subject: [PATCH 002/156] fix(migration): add return type --- src/Appwrite/Migration/Migration.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Migration/Migration.php b/src/Appwrite/Migration/Migration.php index f2d6a8c0a..8d8100ee7 100644 --- a/src/Appwrite/Migration/Migration.php +++ b/src/Appwrite/Migration/Migration.php @@ -26,7 +26,7 @@ abstract class Migration /** * Set project for migration. */ - public function setProject(Document $project, Database $projectDB) + public function setProject(Document $project, Database $projectDB): Migration { $this->project = $project; $this->projectDB = $projectDB; From c51caac494f52b1c30452e0555ab5f0b364cd9e8 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Thu, 14 Jan 2021 13:04:16 +0100 Subject: [PATCH 003/156] refactor(collections): enfore camel case --- app/config/collections.php | 2 +- app/controllers/api/account.php | 6 +++--- app/controllers/api/teams.php | 2 +- app/controllers/api/users.php | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/app/config/collections.php b/app/config/collections.php index a9451d4a5..89236c684 100644 --- a/app/config/collections.php +++ b/app/config/collections.php @@ -228,7 +228,7 @@ $collections = [ [ '$collection' => Database::SYSTEM_COLLECTION_RULES, 'label' => 'Password Update Date', - 'key' => 'password-update', + 'key' => 'passwordUpdate', 'type' => Database::SYSTEM_VAR_TYPE_NUMERIC, 'default' => '', 'required' => true, diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index c28dd19e3..7e0b53866 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -99,7 +99,7 @@ App::post('/v1/account') 'emailVerification' => false, 'status' => Auth::USER_STATUS_UNACTIVATED, 'password' => Auth::passwordHash($password), - 'password-update' => \time(), + 'passwordUpdate' => \time(), 'registration' => \time(), 'reset' => false, 'name' => $name, @@ -511,7 +511,7 @@ App::get('/v1/account/sessions/oauth2/:provider/redirect') 'emailVerification' => true, 'status' => Auth::USER_STATUS_ACTIVATED, // Email should already be authenticated by OAuth2 provider 'password' => Auth::passwordHash(Auth::passwordGenerator()), - 'password-update' => \time(), + 'passwordUpdate' => \time(), 'registration' => \time(), 'reset' => false, 'name' => $name, @@ -1412,7 +1412,7 @@ App::put('/v1/account/recovery') $profile = $projectDB->updateDocument(\array_merge($profile->getArrayCopy(), [ 'password' => Auth::passwordHash($password), - 'password-update' => \time(), + 'passwordUpdate' => \time(), 'emailVerification' => true, ])); diff --git a/app/controllers/api/teams.php b/app/controllers/api/teams.php index 084bdeeb1..5ebeb6366 100644 --- a/app/controllers/api/teams.php +++ b/app/controllers/api/teams.php @@ -324,7 +324,7 @@ App::post('/v1/teams/:teamId/memberships') 'emailVerification' => false, 'status' => Auth::USER_STATUS_UNACTIVATED, 'password' => Auth::passwordHash(Auth::passwordGenerator()), - 'password-update' => \time(), + 'passwordUpdate' => \time(), 'registration' => \time(), 'reset' => false, 'name' => $name, diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index 76e20aad9..a0cff9723 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -62,7 +62,7 @@ App::post('/v1/users') 'emailVerification' => false, 'status' => Auth::USER_STATUS_UNACTIVATED, 'password' => Auth::passwordHash($password), - 'password-update' => \time(), + 'passwordUpdate' => \time(), 'registration' => \time(), 'reset' => false, 'name' => $name, From 328405a774ca60779ebda793041c1560f3c15ae1 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Thu, 14 Jan 2021 13:05:49 +0100 Subject: [PATCH 004/156] fix(migration): move iteration to parent class --- src/Appwrite/Migration/Migration.php | 99 ++++++++---- src/Appwrite/Migration/Version/V04.php | 8 +- src/Appwrite/Migration/Version/V05.php | 210 ++++++++++--------------- src/Appwrite/Migration/Version/V06.php | 26 ++- 4 files changed, 183 insertions(+), 160 deletions(-) diff --git a/src/Appwrite/Migration/Migration.php b/src/Appwrite/Migration/Migration.php index 8d8100ee7..0e7f95abd 100644 --- a/src/Appwrite/Migration/Migration.php +++ b/src/Appwrite/Migration/Migration.php @@ -4,38 +4,83 @@ namespace Appwrite\Migration; use Appwrite\Database\Document; use Appwrite\Database\Database; +use Utopia\CLI\Console; +use Utopia\Exception; abstract class Migration { - protected \PDO $db; + protected \PDO $db; - protected int $limit = 30; - protected int $sum = 30; - protected int $offset = 0; - protected Document $project; - protected Database $projectDB; + protected int $limit = 30; + protected int $sum = 30; + protected int $offset = 0; + protected Document $project; + protected Database $projectDB; - /** - * Migration constructor. - */ - public function __construct(\PDO $db) - { - $this->db = $db; - } + /** + * Migration constructor. + */ + public function __construct(\PDO $db) + { + $this->db = $db; + } - /** - * Set project for migration. - */ - public function setProject(Document $project, Database $projectDB): Migration - { - $this->project = $project; - $this->projectDB = $projectDB; - $this->projectDB->setNamespace('app_'.$project->getId()); - return $this; - } + /** + * Set project for migration. + */ + public function setProject(Document $project, Database $projectDB): Migration + { + $this->project = $project; + $this->projectDB = $projectDB; + $this->projectDB->setNamespace('app_' . $project->getId()); + return $this; + } - /** - * Executes migration for set project. - */ - abstract public function execute(): void; + /** + * Iterates through every document. + * + * @param function(Document): Document $callback + */ + public function forEachDocument(callable $callback) + { + while ($this->sum >= 30) { + $all = $this->projectDB->getCollection([ + 'limit' => $this->limit, + 'offset' => $this->offset, + 'orderType' => 'DESC', + ]); + + $this->sum = \count($all); + + Console::log('Migrating: ' . $this->offset . ' / ' . $this->projectDB->getSum()); + + foreach ($all as $document) { + + $document = call_user_func($callback, $document); + + if (empty($document->getId())) { + throw new Exception('Missing ID'); + } + + try { + $new = $this->projectDB->overwriteDocument($document->getArrayCopy()); + } catch (\Throwable $th) { + var_dump($document); + Console::error('Failed to update document: ' . $th->getMessage()); + continue; + } + + if ($new->getId() !== $document->getId()) { + throw new Exception('Duplication Error'); + } + } + + $this->offset = $this->offset + $this->limit; + } + } + + /** + * Executes migration for set project. + */ + abstract public function execute(): void; } diff --git a/src/Appwrite/Migration/Version/V04.php b/src/Appwrite/Migration/Version/V04.php index f1f879d89..3b13d8920 100644 --- a/src/Appwrite/Migration/Version/V04.php +++ b/src/Appwrite/Migration/Version/V04.php @@ -7,8 +7,8 @@ use Appwrite\Migration\Migration; class V04 extends Migration { - public function execute(): void - { - Console::log('I got nothing to do.'); - } + public function execute(): void + { + Console::log('I got nothing to do.'); + } } diff --git a/src/Appwrite/Migration/Version/V05.php b/src/Appwrite/Migration/Version/V05.php index e8e0d7b4b..c79c059db 100644 --- a/src/Appwrite/Migration/Version/V05.php +++ b/src/Appwrite/Migration/Version/V05.php @@ -5,63 +5,25 @@ namespace Appwrite\Migration\Version; use Appwrite\Migration\Migration; use Utopia\Config\Config; use Utopia\CLI\Console; -use Utopia\Exception; use Appwrite\Database\Database; use Appwrite\Database\Document; class V05 extends Migration { - public function execute(): void - { - $db = $this->db; - $project = $this->project; - $projectDB = $this->projectDB; - Console::log('Migrating project: ' . $project->getAttribute('name') . ' (' . $project->getId() . ')'); + public function execute(): void + { + $db = $this->db; + $project = $this->project; + Console::log('Migrating project: ' . $project->getAttribute('name') . ' (' . $project->getId() . ')'); - // Update all documents $uid -> $id + // Update all documents $uid -> $id - $limit = 30; - $sum = 30; - $offset = 0; + $this->forEachDocument([$this, 'fixDocument']); - while ($sum >= 30) { - $all = $projectDB->getCollection([ - 'limit' => $limit, - 'offset' => $offset, - 'orderType' => 'DESC', - ]); - - $sum = \count($all); - - Console::log('Migrating: ' . $offset . ' / ' . $projectDB->getSum()); - - foreach ($all as $document) { - $document = $this->fixDocument($document); - - if (empty($document->getId())) { - throw new Exception('Missing ID'); - } + $schema = $_SERVER['_APP_DB_SCHEMA'] ?? ''; try { - $new = $projectDB->overwriteDocument($document->getArrayCopy()); - } catch (\Throwable $th) { - var_dump($document); - Console::error('Failed to update document: ' . $th->getMessage()); - continue; - } - - if ($new->getId() !== $document->getId()) { - throw new Exception('Duplication Error'); - } - } - - $offset = $offset + $limit; - } - - $schema = $_SERVER['_APP_DB_SCHEMA'] ?? ''; - - try { - $statement = $db->prepare(" + $statement = $db->prepare(" CREATE TABLE IF NOT EXISTS `template.database.unique` ( `id` int(10) unsigned NOT NULL AUTO_INCREMENT, @@ -76,89 +38,89 @@ class V05 extends Migration ALTER TABLE `{$schema}`.`app_{$project->getId()}.audit.audit` ADD INDEX IF NOT EXISTS `index_1` (`userId` ASC); "); - $statement->closeCursor(); + $statement->closeCursor(); - $statement->execute(); - } catch (\Exception $e) { - Console::error('Failed to alter table for project: ' . $project->getId() . ' with message: ' . $e->getMessage() . '/'); - } - } - - private function fixDocument(Document $document) - { - $providers = Config::getParam('providers'); - - switch ($document->getAttribute('$collection')) { - case Database::SYSTEM_COLLECTION_PROJECTS: - foreach ($providers as $key => $provider) { - if (!empty($document->getAttribute('usersOauth' . \ucfirst($key) . 'Appid'))) { - $document - ->setAttribute('usersOauth2' . \ucfirst($key) . 'Appid', $document->getAttribute('usersOauth' . \ucfirst($key) . 'Appid', '')) - ->removeAttribute('usersOauth' . \ucfirst($key) . 'Appid'); - } - - if (!empty($document->getAttribute('usersOauth' . \ucfirst($key) . 'Secret'))) { - $document - ->setAttribute('usersOauth2' . \ucfirst($key) . 'Secret', $document->getAttribute('usersOauth' . \ucfirst($key) . 'Secret', '')) - ->removeAttribute('usersOauth' . \ucfirst($key) . 'Secret'); - } + $statement->execute(); + } catch (\Exception $e) { + Console::error('Failed to alter table for project: ' . $project->getId() . ' with message: ' . $e->getMessage() . '/'); } - break; - - case Database::SYSTEM_COLLECTION_PROJECTS: - case Database::SYSTEM_COLLECTION_TASKS: - $document->setAttribute('security', ($document->getAttribute('security')) ? true : false); - break; - - case Database::SYSTEM_COLLECTION_USERS: - foreach ($providers as $key => $provider) { - if (!empty($document->getAttribute('oauth' . \ucfirst($key)))) { - $document - ->setAttribute('oauth2' . \ucfirst($key), $document->getAttribute('oauth' . \ucfirst($key), '')) - ->removeAttribute('oauth' . \ucfirst($key)); - } - - if (!empty($document->getAttribute('oauth' . \ucfirst($key) . 'AccessToken'))) { - $document - ->setAttribute('oauth2' . \ucfirst($key) . 'AccessToken', $document->getAttribute('oauth' . \ucfirst($key) . 'AccessToken', '')) - ->removeAttribute('oauth' . \ucfirst($key) . 'AccessToken'); - } - } - - if ($document->getAttribute('confirm', null) !== null) { - $document - ->setAttribute('emailVerification', $document->getAttribute('confirm', $document->getAttribute('emailVerification', false))) - ->removeAttribute('confirm'); - } - break; - - case Database::SYSTEM_COLLECTION_PLATFORMS: - if ($document->getAttribute('url', null) !== null) { - $document - ->setAttribute('hostname', \parse_url($document->getAttribute('url', $document->getAttribute('hostname', '')), PHP_URL_HOST)) - ->removeAttribute('url'); - } - break; } - $document - ->setAttribute('$id', $document->getAttribute('$uid', $document->getAttribute('$id'))) - ->removeAttribute('$uid'); + protected function fixDocument(Document $document) + { + $providers = Config::getParam('providers'); - foreach ($document as &$attr) { // Handle child documents - if ($attr instanceof Document) { - $attr = $this->fixDocument($attr); - } + switch ($document->getAttribute('$collection')) { + case Database::SYSTEM_COLLECTION_PROJECTS: + foreach ($providers as $key => $provider) { + if (!empty($document->getAttribute('usersOauth' . \ucfirst($key) . 'Appid'))) { + $document + ->setAttribute('usersOauth2' . \ucfirst($key) . 'Appid', $document->getAttribute('usersOauth' . \ucfirst($key) . 'Appid', '')) + ->removeAttribute('usersOauth' . \ucfirst($key) . 'Appid'); + } - if (\is_array($attr)) { - foreach ($attr as &$child) { - if ($child instanceof Document) { - $child = $this->fixDocument($child); - } + if (!empty($document->getAttribute('usersOauth' . \ucfirst($key) . 'Secret'))) { + $document + ->setAttribute('usersOauth2' . \ucfirst($key) . 'Secret', $document->getAttribute('usersOauth' . \ucfirst($key) . 'Secret', '')) + ->removeAttribute('usersOauth' . \ucfirst($key) . 'Secret'); + } + } + break; + + case Database::SYSTEM_COLLECTION_PROJECTS: + case Database::SYSTEM_COLLECTION_TASKS: + $document->setAttribute('security', ($document->getAttribute('security')) ? true : false); + break; + + case Database::SYSTEM_COLLECTION_USERS: + foreach ($providers as $key => $provider) { + if (!empty($document->getAttribute('oauth' . \ucfirst($key)))) { + $document + ->setAttribute('oauth2' . \ucfirst($key), $document->getAttribute('oauth' . \ucfirst($key), '')) + ->removeAttribute('oauth' . \ucfirst($key)); + } + + if (!empty($document->getAttribute('oauth' . \ucfirst($key) . 'AccessToken'))) { + $document + ->setAttribute('oauth2' . \ucfirst($key) . 'AccessToken', $document->getAttribute('oauth' . \ucfirst($key) . 'AccessToken', '')) + ->removeAttribute('oauth' . \ucfirst($key) . 'AccessToken'); + } + } + + if ($document->getAttribute('confirm', null) !== null) { + $document + ->setAttribute('emailVerification', $document->getAttribute('confirm', $document->getAttribute('emailVerification', false))) + ->removeAttribute('confirm'); + } + break; + + case Database::SYSTEM_COLLECTION_PLATFORMS: + if ($document->getAttribute('url', null) !== null) { + $document + ->setAttribute('hostname', \parse_url($document->getAttribute('url', $document->getAttribute('hostname', '')), PHP_URL_HOST)) + ->removeAttribute('url'); + } + break; } - } - } - return $document; - } + $document + ->setAttribute('$id', $document->getAttribute('$uid', $document->getAttribute('$id'))) + ->removeAttribute('$uid'); + + foreach ($document as &$attr) { // Handle child documents + if ($attr instanceof Document) { + $attr = $this->fixDocument($attr); + } + + if (\is_array($attr)) { + foreach ($attr as &$child) { + if ($child instanceof Document) { + $child = $this->fixDocument($child); + } + } + } + } + + return $document; + } } diff --git a/src/Appwrite/Migration/Version/V06.php b/src/Appwrite/Migration/Version/V06.php index 33ddafe81..d28d3b652 100644 --- a/src/Appwrite/Migration/Version/V06.php +++ b/src/Appwrite/Migration/Version/V06.php @@ -3,15 +3,31 @@ namespace Appwrite\Migration\Version; use Utopia\CLI\Console; +use Appwrite\Database\Database; +use Appwrite\Database\Document; use Appwrite\Migration\Migration; class V06 extends Migration { - public function execute(): void - { - Console::log('I got nothing to do. Yet.'); + public function execute(): void + { + Console::log('I got nothing to do. Yet.'); - //TODO: migrate new `filter` property + //TODO: migrate new `filter` property + $this->forEachDocument([$this, 'fixDocument']); + } - } + protected function fixDocument(Document $document) + { + switch ($document->getAttribute('$collection')) { + case Database::SYSTEM_COLLECTION_USERS: + if ($document->getAttribute('password-update', null)) { + $document + ->setAttribute('passwordUpdate', $document->getAttribute('password-update', $document->getAttribute('passwordUpdate', ''))) + ->removeAttribute('password-update'); + } + break; + } + return $document; + } } From bc71a1868bd2a0e93828bb3edb2801dbe42df2f7 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Thu, 14 Jan 2021 13:50:26 +0100 Subject: [PATCH 005/156] fix(migration): phpdoc callable --- src/Appwrite/Migration/Migration.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Migration/Migration.php b/src/Appwrite/Migration/Migration.php index 0e7f95abd..51ec1a6e0 100644 --- a/src/Appwrite/Migration/Migration.php +++ b/src/Appwrite/Migration/Migration.php @@ -39,7 +39,7 @@ abstract class Migration /** * Iterates through every document. * - * @param function(Document): Document $callback + * @param callable(Document):Document $callback */ public function forEachDocument(callable $callback) { From d41ee76d3621908d7480b47a748bd5b8fa6953dc Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Thu, 14 Jan 2021 14:12:55 +0100 Subject: [PATCH 006/156] fix(migration): prepare filter migration --- src/Appwrite/Migration/Version/V06.php | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/src/Appwrite/Migration/Version/V06.php b/src/Appwrite/Migration/Version/V06.php index d28d3b652..b9a253701 100644 --- a/src/Appwrite/Migration/Version/V06.php +++ b/src/Appwrite/Migration/Version/V06.php @@ -3,6 +3,7 @@ namespace Appwrite\Migration\Version; use Utopia\CLI\Console; +use Utopia\Config\Config; use Appwrite\Database\Database; use Appwrite\Database\Document; use Appwrite\Migration\Migration; @@ -13,12 +14,13 @@ class V06 extends Migration { Console::log('I got nothing to do. Yet.'); - //TODO: migrate new `filter` property $this->forEachDocument([$this, 'fixDocument']); } protected function fixDocument(Document $document) { + $providers = Config::getParam('providers'); + switch ($document->getAttribute('$collection')) { case Database::SYSTEM_COLLECTION_USERS: if ($document->getAttribute('password-update', null)) { @@ -26,6 +28,26 @@ class V06 extends Migration ->setAttribute('passwordUpdate', $document->getAttribute('password-update', $document->getAttribute('passwordUpdate', ''))) ->removeAttribute('password-update'); } + if($document->getAttribute('prefs', null)) { + //TODO: take care of filter ['json'] + } + break; + case Database::SYSTEM_COLLECTION_WEBHOOKS: + if($document->getAttribute('httpPass', null)) { + //TODO: take care of filter ['encrypt'] + } + break; + case Database::SYSTEM_COLLECTION_TASKS: + if($document->getAttribute('httpPass', null)) { + //TODO: take care of filter ['encrypt'] + } + break; + case Database::SYSTEM_COLLECTION_PROJECTS: + foreach ($providers as $key => $provider) { + if ($document->getAttribute('usersOauth' . \ucfirst($key) . 'Secret')) { + //TODO: take care of filter ['encrypt] + } + } break; } return $document; From 28d4552ee214bf584415debd4daf783161096cf6 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Thu, 14 Jan 2021 14:16:17 +0100 Subject: [PATCH 007/156] fix(migration): missing null value in provider --- src/Appwrite/Migration/Version/V06.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Migration/Version/V06.php b/src/Appwrite/Migration/Version/V06.php index b9a253701..46798b315 100644 --- a/src/Appwrite/Migration/Version/V06.php +++ b/src/Appwrite/Migration/Version/V06.php @@ -44,7 +44,7 @@ class V06 extends Migration break; case Database::SYSTEM_COLLECTION_PROJECTS: foreach ($providers as $key => $provider) { - if ($document->getAttribute('usersOauth' . \ucfirst($key) . 'Secret')) { + if ($document->getAttribute('usersOauth' . \ucfirst($key) . 'Secret', null)) { //TODO: take care of filter ['encrypt] } } From a6a7f896c1cd5076e82cfd8b5062ee3bd4fd8b49 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Thu, 14 Jan 2021 15:03:34 +0100 Subject: [PATCH 008/156] fix(migration): major project iteration bug --- app/tasks/migrate.php | 8 +++++--- src/Appwrite/Migration/Version/V06.php | 13 +++++++------ 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/app/tasks/migrate.php b/app/tasks/migrate.php index c25763dde..7b9b61cc0 100644 --- a/app/tasks/migrate.php +++ b/app/tasks/migrate.php @@ -38,7 +38,7 @@ $cli $migration = new Version\V06($register->get('db')); //TODO: remove hardcoded version and move to dynamic migration - while ($sum >= 30) { + while ($sum > 0) { foreach ($projects as $project) { try { $migration @@ -61,8 +61,10 @@ $cli $sum = \count($projects); $offset = $offset + $limit; $count = $count + $sum; - - Console::log('Fetched ' . $count . '/' . $consoleDB->getSum() . ' projects...'); + + if ($sum > 0) { + Console::log('Fetched ' . $count . '/' . $consoleDB->getSum() . ' projects...'); + } } Console::success('Data Migration Completed'); diff --git a/src/Appwrite/Migration/Version/V06.php b/src/Appwrite/Migration/Version/V06.php index 46798b315..f53313544 100644 --- a/src/Appwrite/Migration/Version/V06.php +++ b/src/Appwrite/Migration/Version/V06.php @@ -12,15 +12,14 @@ class V06 extends Migration { public function execute(): void { - Console::log('I got nothing to do. Yet.'); + $project = $this->project; + Console::log('Migrating project: ' . $project->getAttribute('name') . ' (' . $project->getId() . ')'); $this->forEachDocument([$this, 'fixDocument']); } protected function fixDocument(Document $document) { - $providers = Config::getParam('providers'); - switch ($document->getAttribute('$collection')) { case Database::SYSTEM_COLLECTION_USERS: if ($document->getAttribute('password-update', null)) { @@ -28,21 +27,23 @@ class V06 extends Migration ->setAttribute('passwordUpdate', $document->getAttribute('password-update', $document->getAttribute('passwordUpdate', ''))) ->removeAttribute('password-update'); } - if($document->getAttribute('prefs', null)) { + if ($document->getAttribute('prefs', null)) { //TODO: take care of filter ['json'] } break; case Database::SYSTEM_COLLECTION_WEBHOOKS: - if($document->getAttribute('httpPass', null)) { + if ($document->getAttribute('httpPass', null)) { //TODO: take care of filter ['encrypt'] } break; case Database::SYSTEM_COLLECTION_TASKS: - if($document->getAttribute('httpPass', null)) { + if ($document->getAttribute('httpPass', null)) { //TODO: take care of filter ['encrypt'] } break; case Database::SYSTEM_COLLECTION_PROJECTS: + $providers = Config::getParam('providers'); + foreach ($providers as $key => $provider) { if ($document->getAttribute('usersOauth' . \ucfirst($key) . 'Secret', null)) { //TODO: take care of filter ['encrypt] From d7276aa502a9c5b73bc56b1d7160946e2f7857ba Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Thu, 14 Jan 2021 15:09:48 +0100 Subject: [PATCH 009/156] fix(migration): change loop dependent integers scope --- src/Appwrite/Migration/Migration.php | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/Appwrite/Migration/Migration.php b/src/Appwrite/Migration/Migration.php index 51ec1a6e0..b0e6f4f99 100644 --- a/src/Appwrite/Migration/Migration.php +++ b/src/Appwrite/Migration/Migration.php @@ -12,8 +12,7 @@ abstract class Migration protected \PDO $db; protected int $limit = 30; - protected int $sum = 30; - protected int $offset = 0; + protected Document $project; protected Database $projectDB; @@ -43,16 +42,19 @@ abstract class Migration */ public function forEachDocument(callable $callback) { - while ($this->sum >= 30) { + $sum = 30; + $offset = 0; + + while ($sum >= 30) { $all = $this->projectDB->getCollection([ 'limit' => $this->limit, - 'offset' => $this->offset, + 'offset' => $offset, 'orderType' => 'DESC', ]); - $this->sum = \count($all); + $sum = \count($all); - Console::log('Migrating: ' . $this->offset . ' / ' . $this->projectDB->getSum()); + Console::log('Migrating: ' . $offset . ' / ' . $this->projectDB->getSum()); foreach ($all as $document) { @@ -75,7 +77,7 @@ abstract class Migration } } - $this->offset = $this->offset + $this->limit; + $offset += $this->limit; } } From 3c5d32e5fc8c8a2ae28508c487679c3112a3aa28 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Thu, 14 Jan 2021 15:47:13 +0100 Subject: [PATCH 010/156] fix(migration): migrate encrypt filter --- src/Appwrite/Migration/Version/V06.php | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/Appwrite/Migration/Version/V06.php b/src/Appwrite/Migration/Version/V06.php index f53313544..715508f78 100644 --- a/src/Appwrite/Migration/Version/V06.php +++ b/src/Appwrite/Migration/Version/V06.php @@ -2,11 +2,13 @@ namespace Appwrite\Migration\Version; +use Utopia\App; use Utopia\CLI\Console; use Utopia\Config\Config; use Appwrite\Database\Database; use Appwrite\Database\Document; use Appwrite\Migration\Migration; +use Appwrite\OpenSSL\OpenSSL; class V06 extends Migration { @@ -22,6 +24,8 @@ class V06 extends Migration { switch ($document->getAttribute('$collection')) { case Database::SYSTEM_COLLECTION_USERS: + var_dump($this->applyFilterEncrypt($document->getAttribute("email"))); + if ($document->getAttribute('password-update', null)) { $document ->setAttribute('passwordUpdate', $document->getAttribute('password-update', $document->getAttribute('passwordUpdate', ''))) @@ -53,4 +57,19 @@ class V06 extends Migration } return $document; } + + private function applyFilterEncrypt(String $value) + { + $key = App::getEnv('_APP_OPENSSL_KEY_V1'); + $iv = OpenSSL::randomPseudoBytes(OpenSSL::cipherIVLength(OpenSSL::CIPHER_AES_128_GCM)); + $tag = null; + + return json_encode([ + 'data' => OpenSSL::encrypt($value, OpenSSL::CIPHER_AES_128_GCM, $key, 0, $iv, $tag), + 'method' => OpenSSL::CIPHER_AES_128_GCM, + 'iv' => bin2hex($iv), + 'tag' => bin2hex($tag), + 'version' => '1', + ]); + } } From 970eac94ad54cab6fa38e520107578b31ab4fe30 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Thu, 14 Jan 2021 16:05:27 +0100 Subject: [PATCH 011/156] fix(migration): migrate encrypt filter --- src/Appwrite/Migration/Version/V06.php | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/src/Appwrite/Migration/Version/V06.php b/src/Appwrite/Migration/Version/V06.php index 715508f78..d73887b37 100644 --- a/src/Appwrite/Migration/Version/V06.php +++ b/src/Appwrite/Migration/Version/V06.php @@ -24,8 +24,6 @@ class V06 extends Migration { switch ($document->getAttribute('$collection')) { case Database::SYSTEM_COLLECTION_USERS: - var_dump($this->applyFilterEncrypt($document->getAttribute("email"))); - if ($document->getAttribute('password-update', null)) { $document ->setAttribute('passwordUpdate', $document->getAttribute('password-update', $document->getAttribute('passwordUpdate', ''))) @@ -36,13 +34,9 @@ class V06 extends Migration } break; case Database::SYSTEM_COLLECTION_WEBHOOKS: - if ($document->getAttribute('httpPass', null)) { - //TODO: take care of filter ['encrypt'] - } - break; case Database::SYSTEM_COLLECTION_TASKS: if ($document->getAttribute('httpPass', null)) { - //TODO: take care of filter ['encrypt'] + $document->setAttribute('httpPass', $this->applyFilterEncrypt($document->getAttribute('httpPass'))); } break; case Database::SYSTEM_COLLECTION_PROJECTS: @@ -50,7 +44,7 @@ class V06 extends Migration foreach ($providers as $key => $provider) { if ($document->getAttribute('usersOauth' . \ucfirst($key) . 'Secret', null)) { - //TODO: take care of filter ['encrypt] + $document->getAttribute('usersOauth' . \ucfirst($key) . 'Secret', $this->applyFilterEncrypt($document->getAttribute('usersOauth' . \ucfirst($key) . 'Secret'))); } } break; From 229c649f9a078cc10c9ef41be23003d33bd52185 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Thu, 14 Jan 2021 16:24:01 +0100 Subject: [PATCH 012/156] revert(migration) encryption --- src/Appwrite/Migration/Version/V06.php | 36 -------------------------- 1 file changed, 36 deletions(-) diff --git a/src/Appwrite/Migration/Version/V06.php b/src/Appwrite/Migration/Version/V06.php index d73887b37..253800fc9 100644 --- a/src/Appwrite/Migration/Version/V06.php +++ b/src/Appwrite/Migration/Version/V06.php @@ -2,13 +2,10 @@ namespace Appwrite\Migration\Version; -use Utopia\App; use Utopia\CLI\Console; -use Utopia\Config\Config; use Appwrite\Database\Database; use Appwrite\Database\Document; use Appwrite\Migration\Migration; -use Appwrite\OpenSSL\OpenSSL; class V06 extends Migration { @@ -29,41 +26,8 @@ class V06 extends Migration ->setAttribute('passwordUpdate', $document->getAttribute('password-update', $document->getAttribute('passwordUpdate', ''))) ->removeAttribute('password-update'); } - if ($document->getAttribute('prefs', null)) { - //TODO: take care of filter ['json'] - } - break; - case Database::SYSTEM_COLLECTION_WEBHOOKS: - case Database::SYSTEM_COLLECTION_TASKS: - if ($document->getAttribute('httpPass', null)) { - $document->setAttribute('httpPass', $this->applyFilterEncrypt($document->getAttribute('httpPass'))); - } - break; - case Database::SYSTEM_COLLECTION_PROJECTS: - $providers = Config::getParam('providers'); - - foreach ($providers as $key => $provider) { - if ($document->getAttribute('usersOauth' . \ucfirst($key) . 'Secret', null)) { - $document->getAttribute('usersOauth' . \ucfirst($key) . 'Secret', $this->applyFilterEncrypt($document->getAttribute('usersOauth' . \ucfirst($key) . 'Secret'))); - } - } break; } return $document; } - - private function applyFilterEncrypt(String $value) - { - $key = App::getEnv('_APP_OPENSSL_KEY_V1'); - $iv = OpenSSL::randomPseudoBytes(OpenSSL::cipherIVLength(OpenSSL::CIPHER_AES_128_GCM)); - $tag = null; - - return json_encode([ - 'data' => OpenSSL::encrypt($value, OpenSSL::CIPHER_AES_128_GCM, $key, 0, $iv, $tag), - 'method' => OpenSSL::CIPHER_AES_128_GCM, - 'iv' => bin2hex($iv), - 'tag' => bin2hex($tag), - 'version' => '1', - ]); - } } From 99d5794f1411956521fb1bf378e824d4345816d6 Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Fri, 15 Jan 2021 00:20:31 +0200 Subject: [PATCH 013/156] Deprecated old Nginx stuff --- Dockerfile.nginx | 185 --------------- app/controllers/api/health.php | 4 - bin/start | 37 --- docker-compose.nginx.yml | 215 ----------------- docker/nginx.conf.template | 149 ------------ docker/ssl/cert.pem | 27 --- docker/ssl/key.pem | 52 ----- docker/supervisord.conf | 200 ---------------- docker/www.conf | 412 --------------------------------- 9 files changed, 1281 deletions(-) delete mode 100644 Dockerfile.nginx delete mode 100755 bin/start delete mode 100644 docker-compose.nginx.yml delete mode 100644 docker/nginx.conf.template delete mode 100644 docker/ssl/cert.pem delete mode 100644 docker/ssl/key.pem delete mode 100644 docker/supervisord.conf delete mode 100644 docker/www.conf diff --git a/Dockerfile.nginx b/Dockerfile.nginx deleted file mode 100644 index b7acd23a7..000000000 --- a/Dockerfile.nginx +++ /dev/null @@ -1,185 +0,0 @@ -FROM ubuntu:18.04 AS builder - -LABEL maintainer="team@appwrite.io" - -ARG TESTING=false - -ENV TZ=Asia/Tel_Aviv \ - DEBIAN_FRONTEND=noninteractive \ - PHP_VERSION=7.4 \ - PHP_REDIS_VERSION=5.2.1 - -RUN \ - apt-get update && \ - apt-get install -y --no-install-recommends --no-install-suggests ca-certificates software-properties-common wget git openssl && \ - LC_ALL=C.UTF-8 add-apt-repository -y ppa:ondrej/php && \ - apt-get update && \ - apt-get install -y --no-install-recommends --no-install-suggests make php$PHP_VERSION php$PHP_VERSION-dev zip unzip php$PHP_VERSION-zip && \ - # Redis Extension - wget -q https://github.com/phpredis/phpredis/archive/$PHP_REDIS_VERSION.tar.gz && \ - tar -xf $PHP_REDIS_VERSION.tar.gz && \ - cd phpredis-$PHP_REDIS_VERSION && \ - phpize$PHP_VERSION && \ - ./configure && \ - make && \ - # Composer - wget https://getcomposer.org/composer.phar && \ - chmod +x ./composer.phar && \ - mv ./composer.phar /usr/bin/composer && \ - #Brotli - cd / && \ - git clone https://github.com/eustas/ngx_brotli.git && \ - cd ngx_brotli && git submodule update --init && cd .. - -WORKDIR /usr/local/src/ - -# Updating PHP Dependencies and Auto-loading... - -ENV TESTING=$TESTING - -COPY composer.* /usr/local/src/ - -RUN composer update --ignore-platform-reqs --optimize-autoloader \ - --no-plugins --no-scripts --prefer-dist \ - `if [ "$TESTING" != "true" ]; then echo "--no-dev"; fi` - -FROM ubuntu:18.04 -LABEL maintainer="team@appwrite.io" - -ARG VERSION=dev - -ENV TZ=Asia/Tel_Aviv \ - DEBIAN_FRONTEND=noninteractive \ - PHP_VERSION=7.4 \ - _APP_SERVER=nginx \ - _APP_ENV=production \ - _APP_DOMAIN=localhost \ - _APP_DOMAIN_TARGET=localhost \ - _APP_HOME=https://appwrite.io \ - _APP_EDITION=community \ - _APP_OPTIONS_ABUSE=enabled \ - _APP_OPTIONS_FORCE_HTTPS=disabled \ - _APP_OPENSSL_KEY_V1=your-secret-key \ - _APP_STORAGE_LIMIT=10000000 \ - _APP_STORAGE_ANTIVIRUS=enabled \ - _APP_REDIS_HOST=redis \ - _APP_REDIS_PORT=6379 \ - _APP_DB_HOST=mariadb \ - _APP_DB_PORT=3306 \ - _APP_DB_USER=root \ - _APP_DB_PASS=password \ - _APP_DB_SCHEMA=appwrite \ - _APP_INFLUXDB_HOST=influxdb \ - _APP_INFLUXDB_PORT=8086 \ - _APP_STATSD_HOST=telegraf \ - _APP_STATSD_PORT=8125 \ - _APP_SMTP_HOST=smtp \ - _APP_SMTP_PORT=25 \ - _APP_SETUP=self-hosted \ - _APP_VERSION=$VERSION -#ENV _APP_SMTP_SECURE '' -#ENV _APP_SMTP_USERNAME '' -#ENV _APP_SMTP_PASSWORD '' - -COPY --from=builder /phpredis-5.2.1/modules/redis.so /usr/lib/php/20190902/ -COPY --from=builder /phpredis-5.2.1/modules/redis.so /usr/lib/php/20190902/ -COPY --from=builder /ngx_brotli /ngx_brotli - -RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone - -RUN \ - apt-get update && \ - apt-get install -y --no-install-recommends --no-install-suggests wget ca-certificates software-properties-common build-essential libpcre3-dev zlib1g-dev libssl-dev openssl gnupg htop supervisor && \ - LC_ALL=C.UTF-8 add-apt-repository -y ppa:ondrej/php && \ - add-apt-repository universe && \ - add-apt-repository ppa:certbot/certbot && \ - apt-get update && \ - apt-get install -y --no-install-recommends --no-install-suggests php$PHP_VERSION php$PHP_VERSION-fpm \ - php$PHP_VERSION-mysqlnd php$PHP_VERSION-curl php$PHP_VERSION-imagick php$PHP_VERSION-mbstring php$PHP_VERSION-dom certbot && \ - # Nginx - wget http://nginx.org/download/nginx-1.19.0.tar.gz && \ - tar -xzvf nginx-1.19.0.tar.gz && rm nginx-1.19.0.tar.gz && \ - cd nginx-1.19.0 && \ - ./configure --prefix=/usr/share/nginx \ - --sbin-path=/usr/sbin/nginx \ - --modules-path=/usr/lib/nginx/modules \ - --conf-path=/etc/nginx/nginx.conf \ - --error-log-path=/var/log/nginx/error.log \ - --http-log-path=/var/log/nginx/access.log \ - --pid-path=/run/nginx.pid \ - --lock-path=/var/lock/nginx.lock \ - --user=www-data \ - --group=www-data \ - --build=Ubuntu \ - --with-http_gzip_static_module \ - --with-http_ssl_module \ - --with-http_v2_module \ - --add-module=/ngx_brotli && \ - make && \ - make install && \ - rm -rf ../nginx-1.19.0 && \ - # Redis Extension - echo extension=redis.so >> /etc/php/$PHP_VERSION/fpm/conf.d/redis.ini && \ - echo extension=redis.so >> /etc/php/$PHP_VERSION/cli/conf.d/redis.ini && \ - # Cleanup - cd ../ && \ - apt-get purge -y --auto-remove wget software-properties-common build-essential libpcre3-dev zlib1g-dev libssl-dev gnupg && \ - apt-get clean && \ - rm -rf /ngx_brotli && \ - rm -rf /var/lib/apt/lists/* - -# Set Upload Limit (default to 100MB) -RUN echo "upload_max_filesize = ${_APP_STORAGE_LIMIT}" >> /etc/php/$PHP_VERSION/fpm/conf.d/appwrite.ini -RUN echo "post_max_size = ${_APP_STORAGE_LIMIT}" >> /etc/php/$PHP_VERSION/fpm/conf.d/appwrite.ini -RUN echo "opcache.preload_user=www-data" >> /etc/php/$PHP_VERSION/fpm/conf.d/appwrite.ini -RUN echo "opcache.preload=/usr/src/code/app/preload.php" >> /etc/php/$PHP_VERSION/fpm/conf.d/appwrite.ini -RUN echo "opcache.enable_cli = 1" >> /etc/php/$PHP_VERSION/fpm/conf.d/appwrite.ini - -# Add logs file -RUN echo "" >> /var/log/appwrite.log - -# Nginx Configuration (with self-signed ssl certificates) -COPY ./docker/nginx.conf.template /etc/nginx/nginx.conf.template -COPY ./docker/ssl/cert.pem /etc/nginx/ssl/cert.pem -COPY ./docker/ssl/key.pem /etc/nginx/ssl/key.pem - -# PHP Configuration -RUN mkdir -p /var/run/php -COPY ./docker/www.conf /etc/php/$PHP_VERSION/fpm/pool.d/www.conf - -# Add PHP Source Code -COPY ./app /usr/src/code/app -COPY ./bin /usr/local/bin -COPY ./docs /usr/src/code/docs -COPY ./public /usr/src/code/public -COPY ./src /usr/src/code/src -COPY --from=builder /usr/local/src/vendor /usr/src/code/vendor - -RUN mkdir -p /storage/uploads && \ - mkdir -p /storage/cache && \ - mkdir -p /storage/config && \ - mkdir -p /storage/certificates && \ - mkdir -p /storage/functions && \ - chown -Rf www-data.www-data /storage/uploads && chmod -Rf 0755 /storage/uploads && \ - chown -Rf www-data.www-data /storage/cache && chmod -Rf 0755 /storage/cache && \ - chown -Rf www-data.www-data /storage/config && chmod -Rf 0755 /storage/config && \ - chown -Rf www-data.www-data /storage/certificates && chmod -Rf 0755 /storage/certificates && \ - chown -Rf www-data.www-data /storage/functions && chmod -Rf 0755 /storage/functions - -# Supervisord Conf -COPY ./docker/supervisord.conf /etc/supervisord.conf - -# Executables -RUN chmod +x /usr/local/bin/start -RUN chmod +x /usr/local/bin/doctor -RUN chmod +x /usr/local/bin/migrate -RUN chmod +x /usr/local/bin/test - -# Letsencrypt Permissions -RUN mkdir -p /etc/letsencrypt/live/ && chmod -Rf 755 /etc/letsencrypt/live/ - -EXPOSE 80 - -WORKDIR /usr/src/code - -CMD ["/bin/bash", "/usr/local/bin/start"] diff --git a/app/controllers/api/health.php b/app/controllers/api/health.php index 74b17f241..21e9f2c94 100644 --- a/app/controllers/api/health.php +++ b/app/controllers/api/health.php @@ -287,10 +287,6 @@ App::get('/v1/health/stats') // Currently only used internally $response ->json([ - 'server' => [ - 'name' => 'nginx', - 'version' => \shell_exec('nginx -v 2>&1'), - ], 'storage' => [ 'used' => Storage::human($device->getDirectorySize($device->getRoot().'/')), 'partitionTotal' => Storage::human($device->getPartitionTotalSpace()), diff --git a/bin/start b/bin/start deleted file mode 100755 index 7dd4ecddc..000000000 --- a/bin/start +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/sh - -export PHP_VERSION=$PHP_VERSION - -chown -Rf www-data.www-data /usr/src/code/ - -sed 's/%_APP_STORAGE_LIMIT%/'$_APP_STORAGE_LIMIT'/g' /etc/nginx/nginx.conf.template > /etc/nginx/nginx.conf - -# Function to update the fpm configuration to make the service environment variables available -function setEnvironmentVariable() { - if [ -z "$2" ]; then - echo "Environment variable '$1' not set." - return - fi - - # Check whether variable already exists - if ! grep -q "\[$1\]" /etc/php/$PHP_VERSION/fpm/pool.d/www.conf; then - # Add variable - echo "env[$1] = $2" >> /etc/php/$PHP_VERSION/fpm/pool.d/www.conf - fi - - # Reset variable - # sed -i "s/^env\[$1.*/env[$1] = $2/g" /etc/php/$PHP_VERSION/fpm/pool.d/www.conf -} - -# Grep for variables that look like MySQL (APP_) -for _curVar in $(env | grep _APP_ | awk -F = '{print $1}');do - # awk has split them by the equals sign - # Pass the name and value to our function - setEnvironmentVariable ${_curVar} ${!_curVar} -done - -# Init server settings -php /usr/src/code/app/tasks/init.php ssl - -# Start supervisord and services -/usr/bin/supervisord -n -c /etc/supervisord.conf diff --git a/docker-compose.nginx.yml b/docker-compose.nginx.yml deleted file mode 100644 index 9388c9939..000000000 --- a/docker-compose.nginx.yml +++ /dev/null @@ -1,215 +0,0 @@ -version: '3' - -services: - traefik: - image: traefik:v2.2 - container_name: appwrite-traefik - command: - - --log.level=DEBUG - - --api.insecure=true - - --providers.file.directory=/storage/config - - --providers.file.watch=true - - --providers.docker=true - - --entrypoints.web.address=:80 - - --entrypoints.websecure.address=:443 - - --accesslog=true - restart: unless-stopped - ports: - - 80:80 - - 443:443 - - 8080:8080 - volumes: - - /var/run/docker.sock:/var/run/docker.sock - - appwrite-config:/storage/config:ro - - appwrite-certificates:/storage/certificates:ro - depends_on: - - appwrite - networks: - - gateway - - appwrite - - appwrite: - container_name: appwrite - build: - context: . - args: - - TESTING=true - - VERSION=dev - restart: unless-stopped - networks: - - appwrite - labels: - - traefik.http.routers.appwrite.rule=PathPrefix(`/`) - - traefik.http.routers.appwrite-secure.rule=PathPrefix(`/`) - - traefik.http.routers.appwrite-secure.tls=true - volumes: - - appwrite-uploads:/storage/uploads:rw - - appwrite-cache:/storage/cache:rw - - appwrite-config:/storage/config:rw - - appwrite-certificates:/storage/certificates:rw - - appwrite-functions:/storage/functions:rw - - ./phpunit.xml:/usr/src/code/phpunit.xml - - ./tests:/usr/src/code/tests - - ./app:/usr/src/code/app - # - ./vendor:/usr/src/code/vendor - - ./docs:/usr/src/code/docs - - ./public:/usr/src/code/public - - ./src:/usr/src/code/src - ports: - - 9501:80 - depends_on: - - mariadb - - redis - # - smtp - - clamav - - influxdb - - telegraf - - maildev - environment: - #- _APP_ENV=production - - _APP_ENV=development - - _APP_OPTIONS_ABUSE=disabled - - _APP_OPTIONS_FORCE_HTTPS=disabled - - _APP_OPENSSL_KEY_V1=your-secret-key - - _APP_DOMAIN=demo.appwrite.io - - _APP_DOMAIN_TARGET=demo.appwrite.io - - _APP_REDIS_HOST=redis - - _APP_REDIS_PORT=6379 - - _APP_DB_HOST=mariadb - - _APP_DB_PORT=3306 - - _APP_DB_SCHEMA=appwrite - - _APP_DB_USER=user - - _APP_DB_PASS=password - - _APP_INFLUXDB_HOST=influxdb - - _APP_INFLUXDB_PORT=8086 - - _APP_STATSD_HOST=telegraf - - _APP_STATSD_PORT=8125 - - _APP_SMTP_HOST=maildev - - _APP_SMTP_PORT=25 - - mariadb: - image: appwrite/mariadb:1.2.0 # fix issues when upgrading using: mysql_upgrade -u root -p - container_name: appwrite-mariadb - restart: unless-stopped - networks: - - appwrite - volumes: - - appwrite-mariadb:/var/lib/mysql:rw - ports: - - "3306:3306" - environment: - - MYSQL_ROOT_PASSWORD=rootsecretpassword - - MYSQL_DATABASE=appwrite - - MYSQL_USER=user - - MYSQL_PASSWORD=password - command: 'mysqld --innodb-flush-method=fsync' - - maildev: - image: djfarrelly/maildev - container_name: appwrite-maildev - restart: unless-stopped - ports: - - '1080:80' - networks: - - appwrite - - # smtp: - # image: appwrite/smtp:1.0.1 - # container_name: appwrite-smtp - # restart: unless-stopped - # networks: - # - appwrite - # environment: - # - MAILNAME=appwrite - # - RELAY_NETWORKS=:192.168.0.0/24:10.0.0.0/16 - - redis: - image: redis:5.0 - container_name: appwrite-redis - restart: unless-stopped - networks: - - appwrite - volumes: - - appwrite-redis:/data:rw - - clamav: - image: appwrite/clamav:1.2.0 - container_name: appwrite-clamav - restart: unless-stopped - networks: - - appwrite - volumes: - - appwrite-uploads:/storage/uploads - - influxdb: - image: influxdb:1.6 - container_name: appwrite-influxdb - restart: unless-stopped - networks: - - appwrite - volumes: - - appwrite-influxdb:/var/lib/influxdb:rw - - telegraf: - image: appwrite/telegraf:1.0.0 - container_name: appwrite-telegraf - restart: unless-stopped - networks: - - appwrite - - # redis-commander: - # image: rediscommander/redis-commander:latest - # restart: unless-stopped - # networks: - # - appwrite - # environment: - # - REDIS_HOSTS=redis - # ports: - # - "8081:8081" - - # resque: - # image: registry.gitlab.com/appwrite/appwrite/resque-web:v1.0.2 - # restart: unless-stopped - # networks: - # - appwrite - # ports: - # - "5678:5678" - # environment: - # - RESQUE_WEB_HOST=redis - # - RESQUE_WEB_PORT=6379 - # - RESQUE_WEB_HTTP_BASIC_AUTH_USER=user - # - RESQUE_WEB_HTTP_BASIC_AUTH_PASSWORD=password - - # chronograf: - # image: chronograf:1.5 - # container_name: appwrite-chronograf - # restart: unless-stopped - # networks: - # - appwrite - # volumes: - # - appwrite-chronograf:/var/lib/chronograf - # ports: - # - "8888:8888" - # environment: - # - INFLUXDB_URL=http://influxdb:8086 - # - KAPACITOR_URL=http://kapacitor:9092 - # - AUTH_DURATION=48h - # - TOKEN_SECRET=duperduper5674829!jwt - # - GH_CLIENT_ID=d86f7145a41eacfc52cc - # - GH_CLIENT_SECRET=9e0081062367a2134e7f2ea95ba1a32d08b6c8ab - # - GH_ORGS=appwrite - -networks: - gateway: - appwrite: - -volumes: - appwrite-mariadb: - appwrite-redis: - appwrite-cache: - appwrite-uploads: - appwrite-certificates: - appwrite-functions: - appwrite-influxdb: - appwrite-chronograf: - appwrite-config: diff --git a/docker/nginx.conf.template b/docker/nginx.conf.template deleted file mode 100644 index d53dd7f77..000000000 --- a/docker/nginx.conf.template +++ /dev/null @@ -1,149 +0,0 @@ -user www-data; -worker_processes auto; -pid /run/nginx.pid; -daemon off; - -events { - worker_connections 2048; - - # multi_accept on; -} - -http { - # Basic Settings - sendfile on; - tcp_nopush on; - tcp_nodelay on; - keepalive_timeout 65; - types_hash_max_size 2048; - client_max_body_size %_APP_STORAGE_LIMIT%; - - # server_names_hash_bucket_size 64; - # server_name_in_redirect off; - include /etc/nginx/mime.types; - default_type application/octet-stream; - - # SSL Settings - #ssl_protocols TLSv1 TLSv1.1 TLSv1.2; # Dropping SSLv3, ref: POODLE - #ssl_prefer_server_ciphers on; - - ssl_protocols TLSv1 TLSv1.1 TLSv1.2; - ssl_ciphers 'kEECDH+ECDSA+AES128 kEECDH+ECDSA+AES256 kEECDH+AES128 kEECDH+AES256 kEDH+AES128 kEDH+AES256 DES-CBC3-SHA +SHA !aNULL !eNULL !LOW !kECDH !DSS !MD5 !EXP !PSK !SRP !CAMELLIA !SEED'; - ssl_prefer_server_ciphers off; - - # Logging Settings - access_log /var/log/nginx/access.log; - error_log /var/log/nginx/error.log; - - # Gzip Settings - gzip on; - gzip_disable "msie6"; - - gzip_vary on; - gzip_proxied any; - gzip_comp_level 6; - gzip_buffers 16 8k; - gzip_http_version 1.1; - gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript image/svg+xml; - - # Brotli Settings - brotli on; - brotli_comp_level 5; - brotli_static on; - brotli_types application/atom+xml application/javascript application/json application/rss+xml - application/vnd.ms-fontobject application/x-font-opentype application/x-font-truetype - application/x-font-ttf application/x-javascript application/xhtml+xml application/xml - font/eot font/opentype font/otf font/truetype image/svg+xml image/vnd.microsoft.icon - image/x-icon image/x-win-bitmap text/css text/javascript text/plain text/xml; - - # Virtual Host Configs - server { - listen 80; ## listen for ipv4; this line is default and implied - listen [::]:80 ipv6only=on; ## listen for ipv6 - listen 443 default ssl http2; - - #ssl on; - ssl_certificate /etc/nginx/ssl/cert.pem; - ssl_certificate_key /etc/nginx/ssl/key.pem; - - root /usr/src/code/public; - index index.php index.html index.htm; - - server_tokens off; - - # Make site accessible from http://localhost/ - #server_name localhost; - - # Disable sendfile as per https://docs.vagrantup.com/v2/synced-folders/virtualbox.html - sendfile off; - - # Add stdout logging - - #error_log /dev/stdout info; - #access_log /dev/stdout; - - access_log off; - - location / { - # First attempt to serve request as file, then - # as directory, then fall back to index.html - try_files $uri $uri/ /index.php?q=$uri&$args; - - } - - # Media: images, icons, video, audio, HTC - location ~* \.(?:ico|cur|gz|svg|svgz|mp4|ogg|woff|woff2|ogv|webm|htc)$ { - expires 1M; - access_log off; - add_header Cache-Control "public"; - } - - # CSS and JavaScript - location ~* \.(?:css|js)$ { - expires 1y; - access_log off; - add_header Cache-Control "public"; - } - - location /images { - expires 1y; - access_log off; - add_header Cache-Control "public"; - } - - location /favicon.png { - expires 1y; - access_log off; - add_header Cache-Control "public"; - } - - #error_page 404 /404.html; - - # redirect server error pages to the static page /50x.html - error_page 500 502 503 504 /50x.html; - location = /50x.html { - root /usr/src/code; - } - - # pass the PHP scripts to FastCGI server listening on 127.0.0.1:9000 - location ~ \.php$ { - try_files $uri =404; - fastcgi_split_path_info ^(.+\.php)(/.+)$; - #fastcgi_pass unix:/var/run/php5-fpm.sock; - fastcgi_pass 127.0.0.1:9000; - fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name; - fastcgi_param SCRIPT_NAME $fastcgi_script_name; - fastcgi_param HTTP_IF_NONE_MATCH $http_if_none_match; - fastcgi_param HTTP_IF_MODIFIED_SINCE $http_if_modified_since; - fastcgi_read_timeout 600; - fastcgi_index index.php; - include fastcgi_params; - } - - # deny access to . files, for security - location ~ /\.(?!well-known).* { - #log_not_found off; - deny all; - } - } -} diff --git a/docker/ssl/cert.pem b/docker/ssl/cert.pem deleted file mode 100644 index 81b724364..000000000 --- a/docker/ssl/cert.pem +++ /dev/null @@ -1,27 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIEpDCCAowCCQDLt1wOwov7hTANBgkqhkiG9w0BAQsFADAUMRIwEAYDVQQDDAls -b2NhbGhvc3QwHhcNMTkxMTA5MTgyNDA3WhcNMjAxMTA4MTgyNDA3WjAUMRIwEAYD -VQQDDAlsb2NhbGhvc3QwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQCp -nPP8ck7HKPVorhtIZfxY8Mf+rnhIhenRMzDTmGtq53dhbg4I1kGcy8WZkyh6QaxQ -PyrNKZXBAQtELSWGgjL3PnVsKWVtQT2YzBSIhHTXct6RbYpd2yLmK77IOmu0DtWF -0QX5TxEB/ryDn92uGxIlKptDxqLrsbN1GhZYkXn0jVp6jIqC32YnxJVItzhSqCns -eJrF96XeXZKlN5TmSbpAwbjyZCkCCWi8q5CLpjrhFdud754pmuiJ03xQn9LXhvIx -OzQ8jPiINr+cR19sZHiAwOdb028gjL8VYdvAiOiJ85UwPwPBeCpJkS4FVWxDpBhQ -XjUGbr5YiACcUNenoEjbSut80VhyAqHVPtEY4WMFclua6vuszGQW4cyhWLKO2pN2 -lno0EakXDZT9P53OKB1gFDvpyK3LmD6B4OvddIBCbSYgAotkcojOlcHVDYJUVcN6 -VgiJQann/QLQXcgu8K0CUY4hVvtHV7RlfMc1QcOote5lpUWDpByYWw8TAUHa7MGX -VjNu4q//w3SVRW1RyxVjtgOznLvAM+kFZFV3epS9A9zyk0XyxDuqjz8oLQwKhzms -/ORaDa7WPlqyJ9mkwRWgs+zthFp5fcMJ4onwXqR9Yem4UMsugBEFFn/3kto2ZsAg -DZjrVnlWOpee645S3GRtNjpu0LFqIfYeuZmowDgYQQIDAQABMA0GCSqGSIb3DQEB -CwUAA4ICAQAbSqv901fqZn2CiSVHP7hZkbPjR3y4gMh5U5UlRviG9pu7ljN7x8Sa -XKPT6AYvDFCe3Xux0EQO2nYVkB8jkEyztZ7DjkF36ww52k/aGD+aQAahqzAgumVK -JsQ4nqek8E1FFquX4uoiVZwulWGQE5XDTJBOzLLHvEmk1xKY1AJ9eEUiUm5eKEvM -kVgQo9nRdkwqWOfV2qmYHQ5co6wpHWL9Vl+jeODeonBmsVcaYXc32bOvtKfJL2rW -8IPrvCEieFlcrK8bnjMPZe1rOichri31nsHfrO8LZFu+ZSU2xsjd4Ao0Fm99/27X -rm+e0XsHpKDok+nYUZ3O8cA16fcn2uZulbihSZDbtaxFJvzHDoBl/reJhVKsNVb8 -f99ygx1435GG9NUMTHJtlMel/vu6uujIvFVIfd9Dl3anjaPNBnhuXQCaZUISRMhF -jAiajQJzSzJknEDss+G+WHTbPip7xRl1L43AHdIqlFRwPVRV7pQWi99+2YOtYYo4 -YZmqksQeBIi8gS1T7sDIheSkvr9nr40W4ez6RIlgNYFm+RvVue1tQJNoZYH+6lvT -Qoe+YOSaXotOV9mBd8ffH84bYoMvKooldRE4q3azunCl3HDcZcjarrCFmqzIDzQw -yAlhTb8+aXDY2EHCYBmbwV9AcgPtQ5LuCPeAsh+g9pNhaqF/f/ljLA== ------END CERTIFICATE----- diff --git a/docker/ssl/key.pem b/docker/ssl/key.pem deleted file mode 100644 index 66510fc70..000000000 --- a/docker/ssl/key.pem +++ /dev/null @@ -1,52 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQCpnPP8ck7HKPVo -rhtIZfxY8Mf+rnhIhenRMzDTmGtq53dhbg4I1kGcy8WZkyh6QaxQPyrNKZXBAQtE -LSWGgjL3PnVsKWVtQT2YzBSIhHTXct6RbYpd2yLmK77IOmu0DtWF0QX5TxEB/ryD -n92uGxIlKptDxqLrsbN1GhZYkXn0jVp6jIqC32YnxJVItzhSqCnseJrF96XeXZKl -N5TmSbpAwbjyZCkCCWi8q5CLpjrhFdud754pmuiJ03xQn9LXhvIxOzQ8jPiINr+c -R19sZHiAwOdb028gjL8VYdvAiOiJ85UwPwPBeCpJkS4FVWxDpBhQXjUGbr5YiACc -UNenoEjbSut80VhyAqHVPtEY4WMFclua6vuszGQW4cyhWLKO2pN2lno0EakXDZT9 -P53OKB1gFDvpyK3LmD6B4OvddIBCbSYgAotkcojOlcHVDYJUVcN6VgiJQann/QLQ -Xcgu8K0CUY4hVvtHV7RlfMc1QcOote5lpUWDpByYWw8TAUHa7MGXVjNu4q//w3SV -RW1RyxVjtgOznLvAM+kFZFV3epS9A9zyk0XyxDuqjz8oLQwKhzms/ORaDa7WPlqy -J9mkwRWgs+zthFp5fcMJ4onwXqR9Yem4UMsugBEFFn/3kto2ZsAgDZjrVnlWOpee -645S3GRtNjpu0LFqIfYeuZmowDgYQQIDAQABAoICADrBq2fldVLa5oDX542h/tQU -vUOFzxdYhJI7CIwUfgmvm5R92pDHID2f/Zjg+KG5hGbcKwidgko1AWEhvqElE2DB -G05X3NIHSr5W3DoaoJtOKLn6V3eCBUn1F4cnbc4XYXKU4VvnPv4Q798tD09UA2oq -o1TMR/4cNg239svBwZytJw3TB9ykZTAbkpd5GSLRLIzFjuBLlQM+KSHg6k0Id2Qd -d+NIPUh+V/EcAdvOvxDgUI8axhCloC62u5b2dsTA87+IQeVD9IjDZodN1kmnWHNJ -4BvYV+PPvhY7KzQ8eUnovuLSwYtRBF0t1OJ2ICYif2W/7OCIlpn2qzd7bemcxf/Z -EhZDW856ZluZy5QVAOEWslHCF2di1PoxSJSoyhdHeJHd8QLMUxpKWMAJwakwNJ7W -LximvjCihxq+MoGXhOtQLKFV5YxXqt0/cMaU4QHromInP7o4eSdleQd91W5c5+Fl -4gj9ICHpVXOhbuhO+cx/k53nEMHsyA4XotHgcrF50wUy4Ow9FqaGG0gmL3GEWzbn -S2KUEfPC8nUqezdqtaPpjmQbL4rTp11pYsrN3pd48j+WNodqu2Mi2r3kT3EcvosI -8okAdW82GhMWZjmGn6SLB//RegP7E28sbqEFZFUPvOd2g+Lma/kVAx7qI4/5ndey -pffNnfHgqqNiSe9WsG0BAoIBAQDcKtrKpuBpb3ENZf1Q0EWBi1RveilKGbaHPrq0 -ESiiLfG1nyL5o0LI/LEYspjiad5Z/VDPLpRux7yI2fDHYeDsMvVeHCjvJe2Csw4D -iYSeHGIBMgsMI8573+t0DqFHdfe97Ds1JFflpotnAo8dBwu6bxvXLofCQZuLwa7E -73G515JqK9Koxv32Wp6RnnkKRxRPwFpmjbepwWf2nUvndyy0YjlmpNSsg89Nc2cA -a86k/V2Wt7Nt9I4hPZ9E4CeW2DakJnm5fasBEPiK7qtt2bp+/YPQNACGJarIy+3F -OcRdA0gTDZ9x9H3zHS9G5Dn1Szp3dG7dhVoIMyRILAG5d8yxAoIBAQDFN8h48RPH -K85guCiDrIO1dpG773YFJ/vy6zMXKf8bF4Hw2LaIO1adKvMIIJbpvGcvmFWLsU+I -XbZjeoMe9LH60m0tIqGxvV7SSbHPX/1MmDOoSw83e0x0LOKaWocmorxBhtOdeAB9 -WT5xb7K9exgUKBi2dYLW0P+WUG4n0M4RivB6y1mKIRNtqcaDu0ZbyW+cAF7G4RlM -CFV89dcM36DwvvVe4R7HPSk+tDgm01gocYEntC6K26BeW65k2Sfe9CJAEGazdbak -u8v9PJ55XZWyGkIVpoKF4loF7BYdzoOmgkYM+f0afMyhmBfJ5H6fNFYLgrutmg4b -8YCvaL2D2aiRAoIBAQCnMvxZLgX6zCEE1dFcT+6ZBKCo0BMPLRvK9b6ABQ/gqheH -oETFZFDRpeUwJmGogFHV8WQvEuaygokRPMF4CULw3XotcCE+DIWk3inkUcke8dsT -oVd2brLerBx5VKryRApSd1Y3c1Q1GReAsRbSKomjmcGA1ttOkNh5eCsrb9PkGGwe -qQ0gE47GSedmGv086uHn9uIwQ6uZBUHYrXf5Xi3bB0UkSEUihi8mWF9+mGCkN62d -SgC/nhtZ7xxHCBvImIZWfsmuLltxQdweVkZl9BWHXyt9MCC9v1lFiGkXgFk5ccaI -ga32sn/74swGgEfrmqfaE9gl7qGC3KPPE2xz1yDhAoIBAEoECYT6VUXmtumts+bX -FAdCnKc/07dTrkcY5m/HHyr3w5i0fKzcOEF8IQHn2TuXrdI7BcALp6GyKgVjsVoo -07Mizj6mRLEENVYOumDt0Y6xgJGkue1EpQjk35a2awqhAK5G/5yVsPlaSQkhtp9O -V1cZRU0VBSnB/mpXfUAMKYqD7oTnVI92omgB07MU0e8Yxn5x1SAm0uuqJQtk6HS4 -aRpxUH1vV7HGznfuAzTvFKL5FlPkV6Ndke5X0jefGEugrEoG3cR0ZTumD4TW/1Ll -QI07NZoSh+HfdZHLbPF61AXl1oyANfF+7P2oqyTmUG9HoRNo2S7qJmluVbF/ScD2 -K0ECggEBAJj8ZlacBoTxXy9mf6pMnDpCznzhs1AHQMZ6XIqgW3QoUNkAm4Fnf7G4 -PdhXRAm2NeQ+CSBtmHl7PZHzLlF4RcQnC9DQHy/P+PGfcYfwWNqZyF7pAkwhtFYK -zsxekl6PxjZ6DvclwU3jgQ4OiNElcF2Yjhtwh0nryyz+SktJieFKoWUz1F71tONw -5TNbiKb1sWKzZvvcvyjUdLuMH/HzfNg5ZxxiJHrD0Zga1jLMzZil+eYhIbUuQT9b -IdEYCvfruwoF/SjjPH/aDPu6jZug/iE288Msqm/YGC9uTW1/GLhrpIyWdposa1Jh -Fdo3SqBMHZO0uGFt0FSpZNv0jNUJdB0= ------END PRIVATE KEY----- diff --git a/docker/supervisord.conf b/docker/supervisord.conf deleted file mode 100644 index 4d3361cc9..000000000 --- a/docker/supervisord.conf +++ /dev/null @@ -1,200 +0,0 @@ -[unix_http_server] -file=/tmp/supervisor.sock ; (the path to the socket file) - -[supervisord] -;logfile=/tmp/supervisord.log ; (main log file;default.conf $CWD/supervisord.log) -logfile=/dev/null -logfile_maxbytes=0 ; (max main logfile bytes b4 rotation;default.conf 50MB) -logfile_backups=10 ; (num of main logfile rotation backups;default.conf 10) -loglevel=info ; (log level;default.conf info; others: debug,warn,trace) -pidfile=/tmp/supervisord.pid ; (supervisord pidfile;default.conf supervisord.pid) -nodaemon=false ; (start in foreground if true;default.conf false) -minfds=1024 ; (min. avail startup file descriptors;default.conf 1024) -minprocs=200 ; (min. avail process descriptors;default.conf 200) -user=root ; - -; the below section must remain in the config file for RPC -; (supervisorctl/web interface) to work, additional interfaces may be -; added by defining them in separate rpcinterface: sections -[rpcinterface:supervisor] -supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface - -[supervisorctl] -serverurl=unix:///tmp/supervisor.sock ; use a unix:// URL for a unix socket - -[program:php7-fpm] -command=php-fpm%(ENV_PHP_VERSION)s -F -autostart=true -autorestart=true -priority=5 -stdout_events_enabled=true -stderr_events_enabled=true -startretries=10 -;stdout_logfile=/dev/stdout -;stdout_logfile_maxbytes=0 -stderr_logfile=/dev/stderr -stderr_logfile_maxbytes = 0 - -[program:nginx] -command=nginx -autostart=true -autorestart=true -priority=10 -stdout_events_enabled=true -stderr_events_enabled=true -startretries=10 -;stdout_logfile=/dev/stdout -;stdout_logfile_maxbytes=0 -stderr_logfile=/dev/stderr -stderr_logfile_maxbytes = 0 - -[program:v1-webhooks] -command=php /usr/src/code/vendor/bin/resque -autostart=true -autorestart=true -priority=10 -environment=QUEUE='v1-webhooks',APP_INCLUDE='/usr/src/code/app/workers/webhooks.php',REDIS_BACKEND='%(ENV__APP_REDIS_HOST)s:%(ENV__APP_REDIS_PORT)s' -stdout_events_enabled=true -stderr_events_enabled=true -stopsignal=QUIT -startretries=10 -;stdout_logfile=/dev/stdout -;stdout_logfile_maxbytes=0 -stdout_logfile=/var/log/appwrite.log -stdout_logfile_maxbytes=5000000 -stderr_logfile=/dev/stderr -stderr_logfile_maxbytes = 0 - -[program:v1-mails] -command=php /usr/src/code/vendor/bin/resque -autostart=true -autorestart=true -priority=10 -environment=QUEUE='v1-mails',APP_INCLUDE='/usr/src/code/app/workers/mails.php',REDIS_BACKEND='%(ENV__APP_REDIS_HOST)s:%(ENV__APP_REDIS_PORT)s' -stdout_events_enabled=true -stderr_events_enabled=true -stopsignal=QUIT -startretries=10 -;stdout_logfile=/dev/stdout -;stdout_logfile_maxbytes=0 -stdout_logfile=/var/log/appwrite.log -stdout_logfile_maxbytes=5000000 -stderr_logfile=/dev/stderr -stderr_logfile_maxbytes = 0 - -[program:v1-audits] -command=php /usr/src/code/vendor/bin/resque -autostart=true -autorestart=true -priority=10 -environment=QUEUE='v1-audits',APP_INCLUDE='/usr/src/code/app/workers/audits.php',REDIS_BACKEND='%(ENV__APP_REDIS_HOST)s:%(ENV__APP_REDIS_PORT)s' -stdout_events_enabled=true -stderr_events_enabled=true -stopsignal=QUIT -startretries=10 -;stdout_logfile=/dev/stdout -;stdout_logfile_maxbytes=0 -stdout_logfile=/var/log/appwrite.log -stdout_logfile_maxbytes=5000000 -stderr_logfile=/dev/stderr -stderr_logfile_maxbytes = 0 - -[program:v1-usage] -command=php /usr/src/code/vendor/bin/resque -autostart=true -autorestart=true -priority=10 -environment=QUEUE='v1-usage',APP_INCLUDE='/usr/src/code/app/workers/usage.php',REDIS_BACKEND='%(ENV__APP_REDIS_HOST)s:%(ENV__APP_REDIS_PORT)s' -stdout_events_enabled=true -stderr_events_enabled=true -stopsignal=QUIT -startretries=10 -;stdout_logfile=/dev/stdout -;stdout_logfile_maxbytes=0 -stdout_logfile=/var/log/appwrite.log -stdout_logfile_maxbytes=5000000 -stderr_logfile=/dev/stderr -stderr_logfile_maxbytes = 0 - -[program:v1-tasks] -command=php /usr/src/code/vendor/bin/resque -autostart=true -autorestart=true -priority=10 -environment=QUEUE='v1-tasks',APP_INCLUDE='/usr/src/code/app/workers/tasks.php',REDIS_BACKEND='%(ENV__APP_REDIS_HOST)s:%(ENV__APP_REDIS_PORT)s' -stdout_events_enabled=true -stderr_events_enabled=true -stopsignal=QUIT -startretries=10 -;stdout_logfile=/dev/stdout -;stdout_logfile_maxbytes=0 -stdout_logfile=/var/log/appwrite.log -stdout_logfile_maxbytes=5000000 -stderr_logfile=/dev/stderr -stderr_logfile_maxbytes = 0 - -[program:v1-deletes] -command=php /usr/src/code/vendor/bin/resque -autostart=true -autorestart=true -priority=10 -environment=QUEUE='v1-deletes',APP_INCLUDE='/usr/src/code/app/workers/deletes.php',REDIS_BACKEND='%(ENV__APP_REDIS_HOST)s:%(ENV__APP_REDIS_PORT)s' -stdout_events_enabled=true -stderr_events_enabled=true -stopsignal=QUIT -startretries=10 -;stdout_logfile=/dev/stdout -;stdout_logfile_maxbytes=0 -stdout_logfile=/var/log/appwrite.log -stdout_logfile_maxbytes=5000000 -stderr_logfile=/dev/stderr -stderr_logfile_maxbytes = 0 - -[program:v1-certificates] -command=php /usr/src/code/vendor/bin/resque -autostart=true -autorestart=true -priority=10 -environment=QUEUE='v1-certificates',APP_INCLUDE='/usr/src/code/app/workers/certificates.php',REDIS_BACKEND='%(ENV__APP_REDIS_HOST)s:%(ENV__APP_REDIS_PORT)s' -stdout_events_enabled=true -stderr_events_enabled=true -stopsignal=QUIT -startretries=10 -;stdout_logfile=/dev/stdout -;stdout_logfile_maxbytes=0 -stdout_logfile=/var/log/appwrite.log -stdout_logfile_maxbytes=5000000 -stderr_logfile=/dev/stderr -stderr_logfile_maxbytes = 0 - -[program:v1-functions] -command=php /usr/src/code/vendor/bin/resque -autostart=true -autorestart=true -priority=10 -environment=QUEUE='v1-functions',APP_INCLUDE='/usr/src/code/app/workers/functions.php',REDIS_BACKEND='%(ENV__APP_REDIS_HOST)s:%(ENV__APP_REDIS_PORT)s' -stdout_events_enabled=true -stderr_events_enabled=true -stopsignal=QUIT -startretries=10 -;stdout_logfile=/dev/stdout -;stdout_logfile_maxbytes=0 -stdout_logfile=/var/log/appwrite.log -stdout_logfile_maxbytes=5000000 -stderr_logfile=/dev/stderr -stderr_logfile_maxbytes = 0 - -[program:v1-schedule] -command=php /usr/src/code/vendor/bin/resque-scheduler -autostart=true -autorestart=true -priority=10 -environment=REDIS_BACKEND='%(ENV__APP_REDIS_HOST)s:%(ENV__APP_REDIS_PORT)s',RESQUE_PHP='/usr/src/code/vendor/autoload.php' -stdout_events_enabled=true -stderr_events_enabled=true -stopsignal=QUIT -startretries=10 -;stdout_logfile=/dev/stdout -;stdout_logfile_maxbytes=0 -stderr_logfile=/dev/stderr -stderr_logfile_maxbytes = 0 diff --git a/docker/www.conf b/docker/www.conf deleted file mode 100644 index 152502e61..000000000 --- a/docker/www.conf +++ /dev/null @@ -1,412 +0,0 @@ -; Start a new pool named 'www'. -; the variable $pool can we used in any directive and will be replaced by the -; pool name ('www' here) -[www] - -; Per pool prefix -; It only applies on the following directives: -; - 'access.log' -; - 'slowlog' -; - 'listen' (unixsocket) -; - 'chroot' -; - 'chdir' -; - 'php_values' -; - 'php_admin_values' -; When not set, the global prefix (or /usr) applies instead. -; Note: This directive can also be relative to the global prefix. -; Default Value: none -;prefix = /path/to/pools/$pool - -; Unix user/group of processes -; Note: The user is mandatory. If the group is not set, the default user's group -; will be used. -user = www-data -group = www-data - -; The address on which to accept FastCGI requests. -; Valid syntaxes are: -; 'ip.add.re.ss:port' - to listen on a TCP socket to a specific IPv4 address on -; a specific port; -; '[ip:6:addr:ess]:port' - to listen on a TCP socket to a specific IPv6 address on -; a specific port; -; 'port' - to listen on a TCP socket to all IPv4 addresses on a -; specific port; -; '[::]:port' - to listen on a TCP socket to all addresses -; (IPv6 and IPv4-mapped) on a specific port; -; '/path/to/unix/socket' - to listen on a unix socket. -; Note: This value is mandatory. -; listen = /var/run/php5-fpm.sock -listen = 127.0.0.1:9000 - -; Set listen(2) backlog. -; Default Value: 65535 (-1 on FreeBSD and OpenBSD) -;listen.backlog = 65535 - -; Set permissions for unix socket, if one is used. In Linux, read/write -; permissions must be set in order to allow connections from a web server. Many -; BSD-derived systems allow connections regardless of permissions. -; Default Values: user and group are set as the running user -; mode is set to 0660 -listen.owner = www-data -listen.group = www-data -;listen.mode = 0660 -; When POSIX Access Control Lists are supported you can set them using -; these options, value is a comma separated list of user/group names. -; When set, listen.owner and listen.group are ignored -;listen.acl_users = -;listen.acl_groups = - -; List of addresses (IPv4/IPv6) of FastCGI clients which are allowed to connect. -; Equivalent to the FCGI_WEB_SERVER_ADDRS environment variable in the original -; PHP FCGI (5.2.2+). Makes sense only with a tcp listening socket. Each address -; must be separated by a comma. If this value is left blank, connections will be -; accepted from any ip address. -; Default Value: any -;listen.allowed_clients = 127.0.0.1 - -; Specify the nice(2) priority to apply to the pool processes (only if set) -; The value can vary from -19 (highest priority) to 20 (lower priority) -; Note: - It will only work if the FPM master process is launched as root -; - The pool processes will inherit the master process priority -; unless it specified otherwise -; Default Value: no set -; process.priority = -19 - -; Choose how the process manager will control the number of child processes. -; Possible Values: -; static - a fixed number (pm.max_children) of child processes; -; dynamic - the number of child processes are set dynamically based on the -; following directives. With this process management, there will be -; always at least 1 children. -; pm.max_children - the maximum number of children that can -; be alive at the same time. -; pm.start_servers - the number of children created on startup. -; pm.min_spare_servers - the minimum number of children in 'idle' -; state (waiting to process). If the number -; of 'idle' processes is less than this -; number then some children will be created. -; pm.max_spare_servers - the maximum number of children in 'idle' -; state (waiting to process). If the number -; of 'idle' processes is greater than this -; number then some children will be killed. -; ondemand - no children are created at startup. Children will be forked when -; new requests will connect. The following parameter are used: -; pm.max_children - the maximum number of children that -; can be alive at the same time. -; pm.process_idle_timeout - The number of seconds after which -; an idle process will be killed. -; Note: This value is mandatory. -pm = dynamic - -; The number of child processes to be created when pm is set to 'static' and the -; maximum number of child processes when pm is set to 'dynamic' or 'ondemand'. -; This value sets the limit on the number of simultaneous requests that will be -; served. Equivalent to the ApacheMaxClients directive with mpm_prefork. -; Equivalent to the PHP_FCGI_CHILDREN environment variable in the original PHP -; CGI. The below defaults are based on a server without much resources. Don't -; forget to tweak pm.* to fit your needs. -; Note: Used when pm is set to 'static', 'dynamic' or 'ondemand' -; Note: This value is mandatory. -pm.max_children = 5 - -; The number of child processes created on startup. -; Note: Used only when pm is set to 'dynamic' -; Default Value: min_spare_servers + (max_spare_servers - min_spare_servers) / 2 -pm.start_servers = 2 - -; The desired minimum number of idle server processes. -; Note: Used only when pm is set to 'dynamic' -; Note: Mandatory when pm is set to 'dynamic' -pm.min_spare_servers = 1 - -; The desired maximum number of idle server processes. -; Note: Used only when pm is set to 'dynamic' -; Note: Mandatory when pm is set to 'dynamic' -pm.max_spare_servers = 3 - -; The number of seconds after which an idle process will be killed. -; Note: Used only when pm is set to 'ondemand' -; Default Value: 10s -;pm.process_idle_timeout = 10s; - -; The number of requests each child process should execute before respawning. -; This can be useful to work around memory leaks in 3rd party libraries. For -; endless request processing specify '0'. Equivalent to PHP_FCGI_MAX_REQUESTS. -; Default Value: 0 -;pm.max_requests = 500 - -; The URI to view the FPM status page. If this value is not set, no URI will be -; recognized as a status page. It shows the following informations: -; pool - the name of the pool; -; process manager - static, dynamic or ondemand; -; start time - the date and time FPM has started; -; start since - number of seconds since FPM has started; -; accepted conn - the number of request accepted by the pool; -; listen queue - the number of request in the queue of pending -; connections (see backlog in listen(2)); -; max listen queue - the maximum number of requests in the queue -; of pending connections since FPM has started; -; listen queue len - the size of the socket queue of pending connections; -; idle processes - the number of idle processes; -; active processes - the number of active processes; -; total processes - the number of idle + active processes; -; max active processes - the maximum number of active processes since FPM -; has started; -; max children reached - number of times, the process limit has been reached, -; when pm tries to start more children (works only for -; pm 'dynamic' and 'ondemand'); -; Value are updated in real time. -; Example output: -; pool: www -; process manager: static -; start time: 01/Jul/2011:17:53:49 +0200 -; start since: 62636 -; accepted conn: 190460 -; listen queue: 0 -; max listen queue: 1 -; listen queue len: 42 -; idle processes: 4 -; active processes: 11 -; total processes: 15 -; max active processes: 12 -; max children reached: 0 -; -; By default the status page output is formatted as text/plain. Passing either -; 'html', 'xml' or 'json' in the query string will return the corresponding -; output syntax. Example: -; http://www.foo.bar/status -; http://www.foo.bar/status?json -; http://www.foo.bar/status?html -; http://www.foo.bar/status?xml -; -; By default the status page only outputs short status. Passing 'full' in the -; query string will also return status for each pool process. -; Example: -; http://www.foo.bar/status?full -; http://www.foo.bar/status?json&full -; http://www.foo.bar/status?html&full -; http://www.foo.bar/status?xml&full -; The Full status returns for each process: -; pid - the PID of the process; -; state - the state of the process (Idle, Running, ...); -; start time - the date and time the process has started; -; start since - the number of seconds since the process has started; -; requests - the number of requests the process has served; -; request duration - the duration in µs of the requests; -; request method - the request method (GET, POST, ...); -; request URI - the request URI with the query string; -; content length - the content length of the request (only with POST); -; user - the user (PHP_AUTH_USER) (or '-' if not set); -; script - the main script called (or '-' if not set); -; last request cpu - the %cpu the last request consumed -; it's always 0 if the process is not in Idle state -; because CPU calculation is done when the request -; processing has terminated; -; last request memory - the max amount of memory the last request consumed -; it's always 0 if the process is not in Idle state -; because memory calculation is done when the request -; processing has terminated; -; If the process is in Idle state, then informations are related to the -; last request the process has served. Otherwise informations are related to -; the current request being served. -; Example output: -; ************************ -; pid: 31330 -; state: Running -; start time: 01/Jul/2011:17:53:49 +0200 -; start since: 63087 -; requests: 12808 -; request duration: 1250261 -; request method: GET -; request URI: /test_mem.php?N=10000 -; content length: 0 -; user: - -; script: /home/fat/web/docs/php/test_mem.php -; last request cpu: 0.00 -; last request memory: 0 -; -; Note: There is a real-time FPM status monitoring sample web page available -; It's available in: /usr/share/php5/fpm/status.html -; -; Note: The value must start with a leading slash (/). The value can be -; anything, but it may not be a good idea to use the .php extension or it -; may conflict with a real PHP file. -; Default Value: not set -;pm.status_path = /status - -; The ping URI to call the monitoring page of FPM. If this value is not set, no -; URI will be recognized as a ping page. This could be used to test from outside -; that FPM is alive and responding, or to -; - create a graph of FPM availability (rrd or such); -; - remove a server from a group if it is not responding (load balancing); -; - trigger alerts for the operating team (24/7). -; Note: The value must start with a leading slash (/). The value can be -; anything, but it may not be a good idea to use the .php extension or it -; may conflict with a real PHP file. -; Default Value: not set -;ping.path = /ping - -; This directive may be used to customize the response of a ping request. The -; response is formatted as text/plain with a 200 response code. -; Default Value: pong -;ping.response = pong - -; The access log file -; Default: not set -;access.log = log/$pool.access.log - -; The access log format. -; The following syntax is allowed -; %%: the '%' character -; %C: %CPU used by the request -; it can accept the following format: -; - %{user}C for user CPU only -; - %{system}C for system CPU only -; - %{total}C for user + system CPU (default) -; %d: time taken to serve the request -; it can accept the following format: -; - %{seconds}d (default) -; - %{miliseconds}d -; - %{mili}d -; - %{microseconds}d -; - %{micro}d -; %e: an environment variable (same as $_ENV or $_SERVER) -; it must be associated with embraces to specify the name of the env -; variable. Some exemples: -; - server specifics like: %{REQUEST_METHOD}e or %{SERVER_PROTOCOL}e -; - HTTP headers like: %{HTTP_HOST}e or %{HTTP_USER_AGENT}e -; %f: script filename -; %l: content-length of the request (for POST request only) -; %m: request method -; %M: peak of memory allocated by PHP -; it can accept the following format: -; - %{bytes}M (default) -; - %{kilobytes}M -; - %{kilo}M -; - %{megabytes}M -; - %{mega}M -; %n: pool name -; %o: output header -; it must be associated with embraces to specify the name of the header: -; - %{Content-Type}o -; - %{X-Powered-By}o -; - %{Transfert-Encoding}o -; - .... -; %p: PID of the child that serviced the request -; %P: PID of the parent of the child that serviced the request -; %q: the query string -; %Q: the '?' character if query string exists -; %r: the request URI (without the query string, see %q and %Q) -; %R: remote IP address -; %s: status (response code) -; %t: server time the request was received -; it can accept a strftime(3) format: -; %d/%b/%Y:%H:%M:%S %z (default) -; %T: time the log has been written (the request has finished) -; it can accept a strftime(3) format: -; %d/%b/%Y:%H:%M:%S %z (default) -; %u: remote user -; -; Default: "%R - %u %t \"%m %r\" %s" -;access.format = "%R - %u %t \"%m %r%Q%q\" %s %f %{mili}d %{kilo}M %C%%" - -; The log file for slow requests -; Default Value: not set -; Note: slowlog is mandatory if request_slowlog_timeout is set -;slowlog = log/$pool.log.slow - -; The timeout for serving a single request after which a PHP backtrace will be -; dumped to the 'slowlog' file. A value of '0s' means 'off'. -; Available units: s(econds)(default), m(inutes), h(ours), or d(ays) -; Default Value: 0 -;request_slowlog_timeout = 0 - -; The timeout for serving a single request after which the worker process will -; be killed. This option should be used when the 'max_execution_time' ini option -; does not stop script execution for some reason. A value of '0' means 'off'. -; Available units: s(econds)(default), m(inutes), h(ours), or d(ays) -; Default Value: 0 -;request_terminate_timeout = 0 - -; Set open file descriptor rlimit. -; Default Value: system defined value -;rlimit_files = 1024 - -; Set max core size rlimit. -; Possible Values: 'unlimited' or an integer greater or equal to 0 -; Default Value: system defined value -;rlimit_core = 0 - -; Chroot to this directory at the start. This value must be defined as an -; absolute path. When this value is not set, chroot is not used. -; Note: you can prefix with '$prefix' to chroot to the pool prefix or one -; of its subdirectories. If the pool prefix is not set, the global prefix -; will be used instead. -; Note: chrooting is a great security feature and should be used whenever -; possible. However, all PHP paths will be relative to the chroot -; (error_log, sessions.save_path, ...). -; Default Value: not set -;chroot = - -; Chdir to this directory at the start. -; Note: relative path can be used. -; Default Value: current directory or / when chroot -chdir = / - -; Redirect worker stdout and stderr into main error log. If not set, stdout and -; stderr will be redirected to /dev/null according to FastCGI specs. -; Note: on highloaded environement, this can cause some delay in the page -; process time (several ms). -; Default Value: no -;catch_workers_output = yes - -; Clear environment in FPM workers -; Prevents arbitrary environment variables from reaching FPM worker processes -; by clearing the environment in workers before env vars specified in this -; pool configuration are added. -; Setting to "no" will make all environment variables available to PHP code -; via getenv(), $_ENV and $_SERVER. -; Default Value: yes -;clear_env = no - -; Limits the extensions of the main script FPM will allow to parse. This can -; prevent configuration mistakes on the web server side. You should only limit -; FPM to .php extensions to prevent malicious users to use other extensions to -; exectute php code. -; Note: set an empty value to allow all extensions. -; Default Value: .php -;security.limit_extensions = .php .php3 .php4 .php5 - -; Pass environment variables like LD_LIBRARY_PATH. All $VARIABLEs are taken from -; the current environment. -; Default Value: clean env -;env[HOSTNAME] = $HOSTNAME -;env[PATH] = /usr/local/bin:/usr/bin:/bin -;env[TMP] = /tmp -;env[TMPDIR] = /tmp -;env[TEMP] = /tmp - -; Additional php.ini defines, specific to this pool of workers. These settings -; overwrite the values previously defined in the php.ini. The directives are the -; same as the PHP SAPI: -; php_value/php_flag - you can set classic ini defines which can -; be overwritten from PHP call 'ini_set'. -; php_admin_value/php_admin_flag - these directives won't be overwritten by -; PHP call 'ini_set' -; For php_*flag, valid values are on, off, 1, 0, true, false, yes or no. - -; Defining 'extension' will load the corresponding shared extension from -; extension_dir. Defining 'disable_functions' or 'disable_classes' will not -; overwrite previously defined php.ini values, but will append the new value -; instead. - -; Note: path INI options can be relative and will be expanded with the prefix -; (pool, global or /usr) - -; Default Value: nothing is defined by default except the values in php.ini and -; specified at startup with the -d argument -;php_admin_value[sendmail_path] = /usr/sbin/sendmail -t -i -f www@my.domain.com -;php_flag[display_errors] = off -;php_admin_value[error_log] = /var/log/fpm-php.www.log -;php_admin_flag[log_errors] = on -;php_admin_value[memory_limit] = 32M From b464a761ced95c71288e1dab028a4f0f5e5f094c Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Fri, 15 Jan 2021 00:22:54 +0200 Subject: [PATCH 014/156] Removed old index.php --- public/index.php | 27 --------------------------- 1 file changed, 27 deletions(-) delete mode 100644 public/index.php diff --git a/public/index.php b/public/index.php deleted file mode 100644 index 4b08fcc70..000000000 --- a/public/index.php +++ /dev/null @@ -1,27 +0,0 @@ -run(new Request(), new Response()); From 0ee954bfeb3b988c553bf45b686f52e0091c5b70 Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Fri, 15 Jan 2021 08:02:48 +0200 Subject: [PATCH 015/156] Cleaning workers --- app/workers/audits.php | 2 +- app/workers/certificates.php | 2 +- app/workers/deletes.php | 2 +- app/workers/functions.php | 51 ++++++++---------------------------- app/workers/mails.php | 2 +- app/workers/tasks.php | 2 +- app/workers/usage.php | 2 +- app/workers/webhooks.php | 2 +- 8 files changed, 18 insertions(+), 47 deletions(-) diff --git a/app/workers/audits.php b/app/workers/audits.php index 3f21ba5b6..d6027dd2e 100644 --- a/app/workers/audits.php +++ b/app/workers/audits.php @@ -6,7 +6,7 @@ use Utopia\CLI\Console; require_once __DIR__.'/../init.php'; -\cli_set_process_title('Audits V1 Worker'); +Console::title('Audits V1 Worker'); Console::success(APP_NAME.' audits worker v1 has started'); diff --git a/app/workers/certificates.php b/app/workers/certificates.php index ef478fcde..a072897d8 100644 --- a/app/workers/certificates.php +++ b/app/workers/certificates.php @@ -12,7 +12,7 @@ use Appwrite\Network\Validator\CNAME; require_once __DIR__.'/../init.php'; -\cli_set_process_title('Certificates V1 Worker'); +Console::title('Certificates V1 Worker'); Console::success(APP_NAME.' certificates worker v1 has started'); diff --git a/app/workers/deletes.php b/app/workers/deletes.php index 36d68f0f9..cf9aa2c45 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -15,7 +15,7 @@ use Utopia\Audit\Adapters\MySQL as AuditAdapter; require_once __DIR__.'/../init.php'; -\cli_set_process_title('Deletes V1 Worker'); +Console::title('Deletes V1 Worker'); Console::success(APP_NAME.' deletes worker v1 has started'."\n"); diff --git a/app/workers/functions.php b/app/workers/functions.php index 577e84733..2712dc74f 100644 --- a/app/workers/functions.php +++ b/app/workers/functions.php @@ -1,7 +1,4 @@ Network (docker stats --no-stream --format="{{.NetIO}}" appwrite) - * -> CPU Time - DONE - * -> Invoctions (+1) - DONE - */ - class FunctionsV1 { public $args = []; @@ -380,6 +342,15 @@ class FunctionsV1 unset($list[$container]); } + /** + * Limit CPU Usage - DONE + * Limit Memory Usage - DONE + * Limit Network Usage + * Limit Storage Usage (//--storage-opt size=120m \) + * Make sure no access to redis, mariadb, influxdb or other system services + * Make sure no access to NFS server / storage volumes + * Access Appwrite REST from internal network for improved performance + */ if(!isset($list[$container])) { // Create contianer if not ready $stdout = ''; $stderr = ''; diff --git a/app/workers/mails.php b/app/workers/mails.php index 5431ee922..6f4422b57 100644 --- a/app/workers/mails.php +++ b/app/workers/mails.php @@ -5,7 +5,7 @@ use Utopia\CLI\Console; require_once __DIR__.'/../init.php'; -\cli_set_process_title('Mails V1 Worker'); +Console::title('Mails V1 Worker'); Console::success(APP_NAME.' mails worker v1 has started'."\n"); diff --git a/app/workers/tasks.php b/app/workers/tasks.php index f654748a6..3f2c0957b 100644 --- a/app/workers/tasks.php +++ b/app/workers/tasks.php @@ -11,7 +11,7 @@ use Cron\CronExpression; require_once __DIR__.'/../init.php'; -\cli_set_process_title('Tasks V1 Worker'); +Console::title('Tasks V1 Worker'); Console::success(APP_NAME.' tasks worker v1 has started'); diff --git a/app/workers/usage.php b/app/workers/usage.php index c0486c4bb..c83ae7ae3 100644 --- a/app/workers/usage.php +++ b/app/workers/usage.php @@ -5,7 +5,7 @@ use Utopia\CLI\Console; require_once __DIR__.'/../init.php'; -\cli_set_process_title('Usage V1 Worker'); +Console::title('Usage V1 Worker'); Console::success(APP_NAME.' usage worker v1 has started'); diff --git a/app/workers/webhooks.php b/app/workers/webhooks.php index 76c94a79f..21e88e6b4 100644 --- a/app/workers/webhooks.php +++ b/app/workers/webhooks.php @@ -10,7 +10,7 @@ use Appwrite\Database\Validator\Authorization; require_once __DIR__.'/../init.php'; -\cli_set_process_title('Webhooks V1 Worker'); +Console::title('Webhooks V1 Worker'); Console::success(APP_NAME.' webhooks worker v1 has started'); From 538510d1717cdbcb7d76fe97d0b5fbd34d138140 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Fri, 15 Jan 2021 12:56:03 +0100 Subject: [PATCH 016/156] feat(collections): encrypt api keys --- app/config/collections.php | 1 + 1 file changed, 1 insertion(+) diff --git a/app/config/collections.php b/app/config/collections.php index 89236c684..9170b0757 100644 --- a/app/config/collections.php +++ b/app/config/collections.php @@ -827,6 +827,7 @@ $collections = [ 'type' => Database::SYSTEM_VAR_TYPE_TEXT, 'default' => '', 'required' => false, + 'filter' => ['encrypt'], ], ], ], From bda5e13ac0a079f9c668fe1e710039d1f349d5f1 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Fri, 15 Jan 2021 12:56:39 +0100 Subject: [PATCH 017/156] feat(Database): introduce enabling/disabling filters --- src/Appwrite/Database/Database.php | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/src/Appwrite/Database/Database.php b/src/Appwrite/Database/Database.php index 8610aeac1..c4e1370e1 100644 --- a/src/Appwrite/Database/Database.php +++ b/src/Appwrite/Database/Database.php @@ -57,6 +57,11 @@ class Database */ static protected $filters = []; + /** + * @var bool + */ + static protected $statusFilters = true; + /** * @var array */ @@ -448,6 +453,26 @@ class Database ]; } + /** + * Disable Attribute decoding + * + * @return void + */ + public static function disableFilters(): void + { + self::$statusFilters = false; + } + + /** + * Enable Attribute decoding + * + * @return void + */ + public static function enableFilters(): void + { + self::$statusFilters = true; + } + public function encode(Document $document):Document { $collection = $this->getDocument($document->getCollection(), true , false); @@ -550,6 +575,10 @@ class Database */ static protected function decodeAttribute(string $name, $value) { + if (!self::$statusFilters) { + return $value; + } + if (!isset(self::$filters[$name])) { return $value; throw new Exception('Filter not found'); From bfccaa13629dcb6d960437734e489548d307301b Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Fri, 15 Jan 2021 12:57:32 +0100 Subject: [PATCH 018/156] feat(migration): add api key migration --- src/Appwrite/Migration/Version/V06.php | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/Appwrite/Migration/Version/V06.php b/src/Appwrite/Migration/Version/V06.php index 253800fc9..34620f93c 100644 --- a/src/Appwrite/Migration/Version/V06.php +++ b/src/Appwrite/Migration/Version/V06.php @@ -2,10 +2,13 @@ namespace Appwrite\Migration\Version; + +use Utopia\App; use Utopia\CLI\Console; use Appwrite\Database\Database; use Appwrite\Database\Document; use Appwrite\Migration\Migration; +use Appwrite\OpenSSL\OpenSSL; class V06 extends Migration { @@ -14,7 +17,9 @@ class V06 extends Migration $project = $this->project; Console::log('Migrating project: ' . $project->getAttribute('name') . ' (' . $project->getId() . ')'); + $this->projectDB->disableFilters(); $this->forEachDocument([$this, 'fixDocument']); + $this->projectDB->enableFilters(); } protected function fixDocument(Document $document) @@ -27,6 +32,21 @@ class V06 extends Migration ->removeAttribute('password-update'); } break; + case Database::SYSTEM_COLLECTION_KEYS: + if ($document->getAttribute('secret', null)) { + $key = App::getEnv('_APP_OPENSSL_KEY_V1'); + $iv = OpenSSL::randomPseudoBytes(OpenSSL::cipherIVLength(OpenSSL::CIPHER_AES_128_GCM)); + $tag = null; + + $document->setAttribute('secret', json_encode([ + 'data' => OpenSSL::encrypt($document->getAttribute('secret'), OpenSSL::CIPHER_AES_128_GCM, $key, 0, $iv, $tag), + 'method' => OpenSSL::CIPHER_AES_128_GCM, + 'iv' => bin2hex($iv), + 'tag' => bin2hex($tag), + 'version' => '1', + ])); + } + break; } return $document; } From 8c1371db4d0db2d6dc907ee76d3ed7af89a91ced Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Fri, 15 Jan 2021 16:35:51 +0100 Subject: [PATCH 019/156] chore(changelog): add migration script changes --- CHANGES.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 1418fb453..f682c7dd2 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -39,6 +39,7 @@ - New OAuth adapter for PayPal sandbox (@armino-dev - [#420](https://github.com/appwrite/appwrite/issues/410)) - Introducing new permssion types: role:guest, role:member, role:app - Disabled rate-limits on server side integrations +- Refactored migration script ### User Interface @@ -105,6 +106,7 @@ - Fixed OAuth redirect when using the self-hosted instance default success URL ([#454](https://github.com/appwrite/appwrite/issues/454)) - Fixed bug denying authentication with Github OAuth provider - Fixed a bug making read permission overwrite write permission in some cases +- Fixed consistent property names in databases by enforcing camel case ## Security @@ -113,6 +115,7 @@ - Now using your `_APP_SYSTEM_EMAIL_ADDRESS` as the email address for issuing and renewing SSL certificates - Block iframe access to Appwrite console using the `X-Frame-Options` header. - Fixed `roles` param input validator +- API Keys are now stored encrypted # Version 0.6.2 (PRE-RELEASE) From 5da6b31edb7db94f32ee7244c9c929acd5b13991 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Fri, 15 Jan 2021 16:36:42 +0100 Subject: [PATCH 020/156] style(migration): remove space on concatenation --- app/tasks/migrate.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/tasks/migrate.php b/app/tasks/migrate.php index 7b9b61cc0..f9e92b24b 100644 --- a/app/tasks/migrate.php +++ b/app/tasks/migrate.php @@ -46,7 +46,7 @@ $cli ->execute(); } catch (\Throwable $th) { throw $th; - Console::error('Failed to update project ("' . $project->getId() . '") version with error: ' . $th->getMessage()); + Console::error('Failed to update project ("'.$project->getId().'") version with error: '.$th->getMessage()); } } @@ -54,7 +54,7 @@ $cli 'limit' => $limit, 'offset' => $offset, 'filters' => [ - '$collection=' . Database::SYSTEM_COLLECTION_PROJECTS, + '$collection='.Database::SYSTEM_COLLECTION_PROJECTS, ], ]); @@ -63,7 +63,7 @@ $cli $count = $count + $sum; if ($sum > 0) { - Console::log('Fetched ' . $count . '/' . $consoleDB->getSum() . ' projects...'); + Console::log('Fetched '.$count.'/'.$consoleDB->getSum().' projects...'); } } From f9afa2c95152b15eb079c1c65f249be4fe201c75 Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Fri, 15 Jan 2021 19:48:36 +0200 Subject: [PATCH 021/156] Minor refactoring --- .env | 2 +- app/config/variables.php | 13 +++++++------ app/controllers/general.php | 4 ++-- docker-compose.yml | 23 ++--------------------- tests/e2e/General/HTTPTest.php | 2 +- 5 files changed, 13 insertions(+), 31 deletions(-) diff --git a/.env b/.env index 1cc5b84a3..b6e3f7a2e 100644 --- a/.env +++ b/.env @@ -3,6 +3,7 @@ _APP_ENV=development _APP_SYSTEM_EMAIL_NAME=Appwrite _APP_SYSTEM_EMAIL_ADDRESS=team@appwrite.io _APP_SYSTEM_SECURITY_EMAIL_ADDRESS=security@appwrite.io +_APP_SYSTEM_RESPONSE_FORMAT= _APP_OPTIONS_ABUSE=disabled _APP_OPTIONS_FORCE_HTTPS=disabled _APP_OPENSSL_KEY_V1=your-secret-key @@ -34,5 +35,4 @@ _APP_FUNCTIONS_CPUS=1 _APP_FUNCTIONS_MEMORY=128 _APP_FUNCTIONS_MEMORY_SWAP=128 _APP_MAINTENANCE_INTERVAL=86400 -_APP_SYSTEM_RESPONSE_FORMAT= _APP_USAGE_STATS=enabled \ No newline at end of file diff --git a/app/config/variables.php b/app/config/variables.php index d713acd5b..2189dc264 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -93,6 +93,13 @@ return [ 'required' => false, 'question' => '', ], + [ + 'name' => '_APP_SYSTEM_RESPONSE_FORMAT', + 'description' => '', + 'default' => '', + 'required' => false, + 'question' => '', + ], [ 'name' => '_APP_USAGE_STATS', 'description' => 'This variable allows you to disable the collection and displaying of usage stats. This value is set to \'enabled\' by default, to disable the usage stats set the value to \'disabled\'. When disabled, it\'s recommended to turn off the Worker Usage, Influxdb and Telegraf containers for better resource usage.', @@ -359,10 +366,4 @@ return [ ], ], ], - [ - 'name' => '_APP_SYSTEM_RESPONSE_FORMAT', - 'default' => '', - 'required' => false, - 'question' => '', - ], ]; \ No newline at end of file diff --git a/app/controllers/general.php b/app/controllers/general.php index e718cc9b5..dfb28d50a 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -116,7 +116,7 @@ App::init(function ($utopia, $request, $response, $console, $project, $user, $lo ->addHeader('Server', 'Appwrite') ->addHeader('X-Content-Type-Options', 'nosniff') ->addHeader('Access-Control-Allow-Methods', 'GET, POST, PUT, PATCH, DELETE') - ->addHeader('Access-Control-Allow-Headers', 'Origin, Cookie, Set-Cookie, X-Requested-With, Content-Type, Access-Control-Allow-Origin, Access-Control-Request-Headers, Accept, X-Appwrite-Project, X-Appwrite-Key, X-Appwrite-Locale, X-Appwrite-Mode, X-SDK-Version, Cache-Control, Expires, Pragma') + ->addHeader('Access-Control-Allow-Headers', 'Origin, Cookie, Set-Cookie, X-Requested-With, Content-Type, Access-Control-Allow-Origin, Access-Control-Request-Headers, Accept, X-Appwrite-Project, X-Appwrite-Key, X-Appwrite-Locale, X-Appwrite-Mode, X-Appwrite-JWT, X-SDK-Version, Cache-Control, Expires, Pragma') ->addHeader('Access-Control-Expose-Headers', 'X-Fallback-Cookies') ->addHeader('Access-Control-Allow-Origin', $refDomain) ->addHeader('Access-Control-Allow-Credentials', 'true') @@ -236,7 +236,7 @@ App::options(function ($request, $response) { $response ->addHeader('Server', 'Appwrite') ->addHeader('Access-Control-Allow-Methods', 'GET, POST, PUT, PATCH, DELETE') - ->addHeader('Access-Control-Allow-Headers', 'Origin, Cookie, Set-Cookie, X-Requested-With, Content-Type, Access-Control-Allow-Origin, Access-Control-Request-Headers, Accept, X-Appwrite-Project, X-Appwrite-Key, X-Appwrite-Locale, X-Appwrite-Mode, X-SDK-Version, Cache-Control, Expires, Pragma, X-Fallback-Cookies') + ->addHeader('Access-Control-Allow-Headers', 'Origin, Cookie, Set-Cookie, X-Requested-With, Content-Type, Access-Control-Allow-Origin, Access-Control-Request-Headers, Accept, X-Appwrite-Project, X-Appwrite-Key, X-Appwrite-Locale, X-Appwrite-Mode, X-Appwrite-JWT, X-SDK-Version, Cache-Control, Expires, Pragma, X-Fallback-Cookies') ->addHeader('Access-Control-Expose-Headers', 'X-Fallback-Cookies') ->addHeader('Access-Control-Allow-Origin', $origin) ->addHeader('Access-Control-Allow-Credentials', 'true') diff --git a/docker-compose.yml b/docker-compose.yml index 83024b829..da6de62f3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -21,7 +21,6 @@ services: - --entrypoints.web.address=:80 - --entrypoints.websecure.address=:443 - --accesslog=true - restart: unless-stopped ports: - 80:80 - 443:443 @@ -43,7 +42,6 @@ services: args: - TESTING=true - VERSION=dev - restart: unless-stopped ports: - 9501:80 networks: @@ -80,6 +78,7 @@ services: - _APP_SYSTEM_EMAIL_NAME - _APP_SYSTEM_EMAIL_ADDRESS - _APP_SYSTEM_SECURITY_EMAIL_ADDRESS + - _APP_SYSTEM_RESPONSE_FORMAT - _APP_OPTIONS_ABUSE - _APP_OPTIONS_FORCE_HTTPS - _APP_OPENSSL_KEY_V1 @@ -106,7 +105,6 @@ services: - _APP_STORAGE_LIMIT - _APP_FUNCTIONS_TIMEOUT - _APP_FUNCTIONS_CONTAINERS - - _APP_SYSTEM_RESPONSE_FORMAT - _APP_FUNCTIONS_CPUS - _APP_FUNCTIONS_MEMORY - _APP_FUNCTIONS_MEMORY_SWAP @@ -116,7 +114,6 @@ services: container_name: appwrite-worker-usage build: context: . - restart: unless-stopped networks: - appwrite volumes: @@ -137,7 +134,6 @@ services: container_name: appwrite-worker-audits build: context: . - restart: unless-stopped networks: - appwrite volumes: @@ -161,7 +157,6 @@ services: container_name: appwrite-worker-webhooks build: context: . - restart: unless-stopped networks: - appwrite volumes: @@ -187,7 +182,6 @@ services: container_name: appwrite-worker-tasks build: context: . - restart: unless-stopped networks: - appwrite volumes: @@ -212,7 +206,6 @@ services: container_name: appwrite-worker-deletes build: context: . - restart: unless-stopped networks: - appwrite volumes: @@ -239,7 +232,6 @@ services: container_name: appwrite-worker-certificates build: context: . - restart: unless-stopped networks: - appwrite volumes: @@ -266,7 +258,6 @@ services: container_name: appwrite-worker-functions build: context: . - restart: unless-stopped networks: - appwrite volumes: @@ -274,8 +265,8 @@ services: - appwrite-functions:/storage/functions:rw - /tmp:/tmp:rw - ./app:/usr/src/code/app - - ./docker:/usr/src/code/docker - ./src:/usr/src/code/src + - ./docker:/usr/src/code/docker depends_on: - redis - mariadb @@ -300,7 +291,6 @@ services: container_name: appwrite-worker-mails build: context: . - restart: unless-stopped networks: - appwrite volumes: @@ -327,7 +317,6 @@ services: container_name: appwrite-maintenance build: context: . - restart: unless-stopped networks: - appwrite depends_on: @@ -348,7 +337,6 @@ services: container_name: appwrite-schedule build: context: . - restart: unless-stopped networks: - appwrite volumes: @@ -364,7 +352,6 @@ services: mariadb: image: appwrite/mariadb:1.2.0 # fix issues when upgrading using: mysql_upgrade -u root -p container_name: appwrite-mariadb - restart: unless-stopped networks: - appwrite volumes: @@ -394,7 +381,6 @@ services: redis: image: redis:6.0-alpine container_name: appwrite-redis - restart: unless-stopped networks: - appwrite volumes: @@ -403,7 +389,6 @@ services: clamav: image: appwrite/clamav:1.2.0 container_name: appwrite-clamav - restart: unless-stopped networks: - appwrite volumes: @@ -412,7 +397,6 @@ services: influxdb: image: influxdb:1.8-alpine container_name: appwrite-influxdb - restart: unless-stopped networks: - appwrite volumes: @@ -421,14 +405,12 @@ services: telegraf: image: appwrite/telegraf:1.0.0 container_name: appwrite-telegraf - restart: unless-stopped networks: - appwrite maildev: # used mainly for dev tests image: djfarrelly/maildev container_name: appwrite-maildev - restart: unless-stopped ports: - '9503:80' networks: @@ -437,7 +419,6 @@ services: request-catcher: # used mainly for dev tests image: smarterdm/http-request-catcher container_name: appwrite-request-catcher - restart: unless-stopped ports: - '9504:5000' networks: diff --git a/tests/e2e/General/HTTPTest.php b/tests/e2e/General/HTTPTest.php index 1282a9e48..df234f1e1 100644 --- a/tests/e2e/General/HTTPTest.php +++ b/tests/e2e/General/HTTPTest.php @@ -26,7 +26,7 @@ class HTTPTest extends Scope $this->assertEquals(200, $response['headers']['status-code']); $this->assertEquals('Appwrite', $response['headers']['server']); $this->assertEquals('GET, POST, PUT, PATCH, DELETE', $response['headers']['access-control-allow-methods']); - $this->assertEquals('Origin, Cookie, Set-Cookie, X-Requested-With, Content-Type, Access-Control-Allow-Origin, Access-Control-Request-Headers, Accept, X-Appwrite-Project, X-Appwrite-Key, X-Appwrite-Locale, X-Appwrite-Mode, X-SDK-Version, Cache-Control, Expires, Pragma, X-Fallback-Cookies', $response['headers']['access-control-allow-headers']); + $this->assertEquals('Origin, Cookie, Set-Cookie, X-Requested-With, Content-Type, Access-Control-Allow-Origin, Access-Control-Request-Headers, Accept, X-Appwrite-Project, X-Appwrite-Key, X-Appwrite-Locale, X-Appwrite-Mode, X-Appwrite-JWT, X-SDK-Version, Cache-Control, Expires, Pragma, X-Fallback-Cookies', $response['headers']['access-control-allow-headers']); $this->assertEquals('X-Fallback-Cookies', $response['headers']['access-control-expose-headers']); $this->assertEquals('http://localhost', $response['headers']['access-control-allow-origin']); $this->assertEquals('true', $response['headers']['access-control-allow-credentials']); From 2f5b829eedbfd8ca29b672560aed1617d8c44aa1 Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Fri, 15 Jan 2021 21:12:06 +0200 Subject: [PATCH 022/156] Added missing description --- app/config/variables.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/config/variables.php b/app/config/variables.php index 2189dc264..22c39bf58 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -95,7 +95,8 @@ return [ ], [ 'name' => '_APP_SYSTEM_RESPONSE_FORMAT', - 'description' => '', + 'description' => 'Use this environment variable to set the default Appwrite HTTP response format to support an older version of Appwrite. This option is useful to overcome breaking changes between versions. You can also use the `X-Appwrite-Response-Format` HTTP request header to overwrite the response for a specific request. The var accepts any valid Appwrite version. To use the current version format, leave the value of the variable empty.', + 'introduction' => '0.7.0', 'default' => '', 'required' => false, 'question' => '', From 8f0558cfc1af6a38f6f10bc3a4221bdb3ad51525 Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Sat, 16 Jan 2021 08:07:49 +0200 Subject: [PATCH 023/156] Updated contribution guide --- CONTRIBUTING.md | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 73e03d1e7..a51844a2b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -72,7 +72,23 @@ cd appwrite docker-compose up -d ``` -After finishing the installation process, you can start writing and editing code. To compile new CSS and JS distribution files, use 'less' and 'build' tasks using gulp as a task manager. +### Code Autocompletion + +To get proper autocompletion for all the different functions and classes in the codebase, you'll need to install Appwrite dependencies on your local machine. You can easily do that with PHP's package manager, [Composer](https://getcomposer.org/). If you don't have Composer installed, you can use the Docker Hub image to get the same result: + +```bash +docker run --rm --interactive --tty \ + --volume $PWD:/app \ + composer install +``` + +### User Interface + +Appwrite uses an internal micro-framework called Litespeed.js to build simple UI components in vanilla JS and [less](http://lesscss.org/) for compiling CSS code. To apply any of your changes to the UI, use the `gulp build` or `gulp less` commands, and restart the Appwrite main container to load the new static files to memory using `docker-compose restart appwrite`. + +### Get Started + +After finishing the installation process, you can start writing and editing code. ## Architecture From bb113539e5cc9c164a6251465ae24fa9c193e045 Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Sat, 16 Jan 2021 08:22:35 +0200 Subject: [PATCH 024/156] Typo fix --- app/config/variables.php | 2 +- app/controllers/web/home.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/config/variables.php b/app/config/variables.php index 22c39bf58..6215b18f7 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -95,7 +95,7 @@ return [ ], [ 'name' => '_APP_SYSTEM_RESPONSE_FORMAT', - 'description' => 'Use this environment variable to set the default Appwrite HTTP response format to support an older version of Appwrite. This option is useful to overcome breaking changes between versions. You can also use the `X-Appwrite-Response-Format` HTTP request header to overwrite the response for a specific request. The var accepts any valid Appwrite version. To use the current version format, leave the value of the variable empty.', + 'description' => 'Use this environment variable to set the default Appwrite HTTP response format to support an older version of Appwrite. This option is useful to overcome breaking changes between versions. You can also use the `X-Appwrite-Response-Format` HTTP request header to overwrite the response for a specific request. This variable accepts any valid Appwrite version. To use the current version format, leave the value of the variable empty.', 'introduction' => '0.7.0', 'default' => '', 'required' => false, diff --git a/app/controllers/web/home.php b/app/controllers/web/home.php index 5b4e9b731..8dc976986 100644 --- a/app/controllers/web/home.php +++ b/app/controllers/web/home.php @@ -191,7 +191,7 @@ App::get('/error/:code') $layout ->setParam('title', 'Error'.' - '.APP_NAME) ->setParam('body', $page); - }, ['']); + }); App::get('/specs/:format') ->groups(['web', 'home']) From c8eeaae1e81ef211615658e6b48f61ab18698ec5 Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Sat, 16 Jan 2021 09:08:54 +0200 Subject: [PATCH 025/156] Updated contribution guide --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a51844a2b..e3448a5dc 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -232,7 +232,7 @@ For us to find the right balance, please open an issue explaining your ideas bef This will allow the Appwrite community to have sufficient discussion about the new feature value and how it fits in the product roadmap and vision. -This is also important for the Appwrite lead developers to be able to give technical input and different emphasis regarding the feature design and architecture. +This is also important for the Appwrite lead developers to be able to give technical input and different emphasis regarding the feature design and architecture. Some bigger features might need to go through our [RFC process](https://github.com/appwrite/rfc). ## Build From a66683dd362ea427514ce7afa7d7b5a818f4fa31 Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Sat, 16 Jan 2021 10:20:59 +0200 Subject: [PATCH 026/156] Added authentication flow --- app/config/roles.php | 6 +- docs/specs/authentication.drawio.svg | 159 +++++++ docs/specs/overview.drawio.svg | 652 ++++++++++++++++++++++++++- 3 files changed, 813 insertions(+), 4 deletions(-) create mode 100644 docs/specs/authentication.drawio.svg diff --git a/app/config/roles.php b/app/config/roles.php index 506b2403c..78dd24ad4 100644 --- a/app/config/roles.php +++ b/app/config/roles.php @@ -2,7 +2,7 @@ use Appwrite\Auth\Auth; -$logged = [ +$member = [ 'public', 'home', 'console', @@ -66,7 +66,7 @@ return [ ], Auth::USER_ROLE_MEMBER => [ 'label' => 'Member', - 'scopes' => \array_merge($logged, []), + 'scopes' => \array_merge($member, []), ], Auth::USER_ROLE_ADMIN => [ 'label' => 'Admin', @@ -78,7 +78,7 @@ return [ ], Auth::USER_ROLE_OWNER => [ 'label' => 'Owner', - 'scopes' => \array_merge($logged, $admins, []), + 'scopes' => \array_merge($member, $admins, []), ], Auth::USER_ROLE_APP => [ 'label' => 'Application', diff --git a/docs/specs/authentication.drawio.svg b/docs/specs/authentication.drawio.svg new file mode 100644 index 000000000..c7958fa22 --- /dev/null +++ b/docs/specs/authentication.drawio.svg @@ -0,0 +1,159 @@ + + + + + + + + + +
+
+
+ Session Cookie +
+
+
+
+ + Session Cookie + +
+
+ + + + + + +
+
+
+ Email / Password +
+
+
+
+ + Email / Password + +
+
+ + + + + + +
+
+
+ OAuth Provider +
+
+
+
+ + OAuth Provider + +
+
+ + + + +
+
+
+ Member +
+
+
+
+ + Member + +
+
+ + + + +
+
+
+ App +
+
+
+
+ + App + +
+
+ + + + + + +
+
+
+ JWT +
+
+
+
+ + JWT + +
+
+ + + + + + +
+
+
+ AP Key +
+
+
+
+ + AP Key + +
+
+ + + + +
+
+
+ Guest +
+
+
+
+ + Guest + +
+
+
+ + + + + Viewer does not support full SVG 1.1 + + + +
\ No newline at end of file diff --git a/docs/specs/overview.drawio.svg b/docs/specs/overview.drawio.svg index 7b376f660..5f6c16100 100644 --- a/docs/specs/overview.drawio.svg +++ b/docs/specs/overview.drawio.svg @@ -1 +1,651 @@ -
Web
Web
Flutter
Flutter
iOS
iOS
Android
Android
Servers
Servers
Appwrite
Appwrite
Loadbalancer
Loadbalancer
Console
Console
APIs
APIs
Pub/Sub (Redis)
Pub/Sub (Redis)
Cache (Redis)
Cache (Redis)
Database (MariaDB)
Database (MariaDB)
Users
Users
Account
Account
Teams
Teams
Database
Database
Storage
Storage
Localization
Localization
Avatars
Avatars
Health
Health
SSL Gateway
SSL Gateway
Deletes
Deletes
Security Layer
Security Layer
Usage
Usage
Audits
Audits
Mails
Mails
SMTP
SMTP
Tasks
Tasks
Webhooks
Webhooks
Functions
Functions
Docker
Docker
StatsD (Telegraf)
StatsD (Telegraf)
TimeSeries (InfluxDB)
TimeSeries (InfluxDB)
Certs
Certs
Scheduler
Scheduler
Letsencrypt
Letsencrypt
AntiVirus (ClamAV)
AntiVirus (ClamAV)
Viewer does not support full SVG 1.1
\ No newline at end of file + + + + + + + + + +
+
+
+ Session Cookie +
+
+
+
+ + Session Cookie + +
+
+ + + + + + +
+
+
+ Email / Password +
+
+
+
+ + Email / Password + +
+
+ + + + + + +
+
+
+ OAuth Provider +
+
+
+
+ + OAuth Provider + +
+
+ + + + + +
+
+
+ Granted with +
+
+
+
+ + Granted with + +
+
+ + + + +
+
+
+ Member +
+
+
+
+ + Member + +
+
+ + + + + +
+
+
+ Granted with +
+
+
+
+ + Granted with + +
+
+ + + + +
+
+
+ App +
+
+
+
+ + App + +
+
+ + + + + + +
+
+
+ JWT +
+
+
+
+ + JWT + +
+
+ + + + + + +
+
+
+ AP Key +
+
+
+
+ + AP Key + +
+
+ + + + + +
+
+
+ Granted with +
+
+
+
+ + Granted with + +
+
+ + + + +
+
+
+ Guest +
+
+
+
+ + Guest + +
+
+ + + + + + +
+
+
+ Public Scopes +
+
+
+
+ + Public Scopes + +
+
+ + + + + + +
+
+
+ Member Scopes +
+
+
+
+ + Member Scopes + +
+
+ + + + + + +
+
+
+ Custom Scopes +
+ + (Defined on key creation) + +
+
+
+
+ + Custom Scopes... + +
+
+ + + + +
+
+
+ Scope Validation +
+
+
+
+ + Scope Validation + +
+
+ + + + + + + + + + +
+
+
+ Database +
+ + Each doc has permission + +
+
+
+
+ + Database... + +
+
+ + + + +
+
+
+ Roles Validation +
+
+
+
+ + Roles Validation + +
+
+ + + + + + +
+
+
+ Roles +
+
+
+
+ + Roles + +
+
+ + + + + + +
+
+
+ Wildcard +
+ * +
+
+
+
+ + Wildcard... + +
+
+ + + + + + +
+
+
+ Guset +
+ role:guest +
+
+
+
+ + Guset... + +
+
+ + + + + + +
+
+
+ Member +
+ role:member +
+
+
+
+ + Member... + +
+
+ + + + + + +
+
+
+ App +
+ role:app +
+
+
+
+ + App... + +
+
+ + + + +
+
+
+ User ID +
+ user:[ID] +
+
+
+
+ + User ID... + +
+
+ + + + + + +
+
+
+ Team ID +
+ team:[ID] +
+
+
+
+ + Team ID... + +
+
+ + + + + + +
+
+
+ Team ID + Role +
+ team:[ID]/[ROLE] +
+
+
+
+ + Team ID + Role... + +
+
+ + + + +
+
+
+ Member ID +
+ member:[ID] +
+
+
+
+ + Member ID... + +
+
+ + + + +
+
+
+ Endpoints +
+ + Each endpoint has 1 scope + +
+
+
+
+ + Endpoints... + +
+
+ + + + + + +
+
+
+ Scopes +
+
+
+
+ + Scopes + +
+
+ + + + + + +
+
+
+ public +
+
+
+
+ + public + +
+
+ + + + + + +
+
+
+ account +
+
+
+
+ + account + +
+
+ + + + + + +
+
+
+ files.read +
+
+
+
+ + files.read + +
+
+ + + + +
+
+
+ files.write +
+
+
+
+ + files.write + +
+
+ + + + +
+
+
+ + ... + +
+
+
+
+ + ... + +
+
+
+ + + + + Viewer does not support full SVG 1.1 + + + +
\ No newline at end of file From e9b15d5863f03b8a163176877769f661a7218c83 Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Sat, 16 Jan 2021 10:24:05 +0200 Subject: [PATCH 027/156] Updated spec --- docs/specs/overview.drawio.svg | 46 +++++++++++++++++----------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/docs/specs/overview.drawio.svg b/docs/specs/overview.drawio.svg index 5f6c16100..f520c0c1c 100644 --- a/docs/specs/overview.drawio.svg +++ b/docs/specs/overview.drawio.svg @@ -1,4 +1,4 @@ - + @@ -201,8 +201,8 @@ - - + + @@ -220,8 +220,8 @@ - - + + @@ -239,8 +239,8 @@ - - + + @@ -262,11 +262,11 @@ - + -
+
Scope Validation @@ -274,22 +274,22 @@
- + Scope Validation - - + + - - - + + + -
+
Database @@ -301,16 +301,16 @@
- + Database... - + -
+
Roles Validation @@ -318,7 +318,7 @@
- + Roles Validation @@ -506,11 +506,11 @@ - + -
+
Endpoints @@ -522,7 +522,7 @@
- + Endpoints... From 82742d86566d5d046fd244a5bfcfa3a1360b857a Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Sat, 16 Jan 2021 12:33:01 +0200 Subject: [PATCH 028/156] Updated auth diagram --- docker-compose.yml | 25 +- docs/specs/authentication.drawio.svg | 674 +++++++++++++++++++++++---- 2 files changed, 607 insertions(+), 92 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index da6de62f3..be55f7acb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -357,7 +357,7 @@ services: volumes: - appwrite-mariadb:/var/lib/mysql:rw ports: - - "9502:3306" + - "3306:3306" environment: - MYSQL_ROOT_PASSWORD=password - MYSQL_DATABASE=${_APP_DB_SCHEMA} @@ -408,6 +408,19 @@ services: networks: - appwrite + # Dev Tools Start ------------------------------------------------------------------------------------------ + # + # The Appwrite Team uses the following tools to help debug, monitor and diagnose the Appwrite stack 🪛 + # + # Here is a description of the different tools and why are we using them: + # + # MailCatcher - An SMTP server. Catches all system emails and displays them in a nice UI. + # RequestCatcher - An HTTP server. Catches all system https calls and displays them using a simple HTTP API. Used to debug & tests webhooks and HTTP tasks + # RedisCommander - A nice UI for exploring Redis data + # Resque - A nice UI for exploring Reddis pub/sub, view the different queues workloads, pending and failed tasks + # Chronograf - A nice UI for exploring InfluxDB data + # Webgrind - A nice UI for exploring and debugging code-level stuff + maildev: # used mainly for dev tests image: djfarrelly/maildev container_name: appwrite-maildev @@ -424,6 +437,14 @@ services: networks: - appwrite + adminer: + image: adminer + restart: always + ports: + - 9505:8080 + networks: + - appwrite + # redis-commander: # image: rediscommander/redis-commander:latest # restart: unless-stopped @@ -472,6 +493,8 @@ services: # - './debug:/tmp' # ports: # - '3001:80' + + # Dev Tools End ------------------------------------------------------------------------------------------ networks: gateway: diff --git a/docs/specs/authentication.drawio.svg b/docs/specs/authentication.drawio.svg index c7958fa22..0f665a33e 100644 --- a/docs/specs/authentication.drawio.svg +++ b/docs/specs/authentication.drawio.svg @@ -1,13 +1,13 @@ - + - - - + + + -
+
Session Cookie @@ -15,126 +15,180 @@
- + Session Cookie - - - + + + -
+
-
+
Email / Password
- + Email / Password - - - + + + -
+
-
+
OAuth Provider
- + OAuth Provider + + + + + +
+
+
+ Granted with +
+
+
+
+ + Granted with + +
+
+ + + + +
+
+
+ Member +
+
+
+
+ + Member + +
+
+ + + + + +
+
+
+ Granted with +
+
+
+
+ + Granted with + +
+
+ + + + +
+
+
+ App +
+
+
+
+ + App + +
+
+ + + + + + +
+
+
+ JWT +
+
+
+
+ + JWT + +
+
+ + + + + + +
+
+
+ AP Key +
+
+
+
+ + AP Key + +
+
+ + + + + +
+
+
+ Granted with +
+
+
+
+ + Granted with + +
+
-
-
- Member -
-
-
-
- - Member - -
-
- - - - -
-
-
- App -
-
-
-
- - App - -
-
- - - - - - -
-
-
- JWT -
-
-
-
- - JWT - -
-
- - - - - - -
-
-
- AP Key -
-
-
-
- - AP Key - -
-
- - - - -
Guest @@ -142,11 +196,449 @@
- + Guest + + + + + + +
+
+
+ Public Scopes +
+
+
+
+ + Public Scopes + +
+
+ + + + + + +
+
+
+ Member Scopes +
+
+
+
+ + Member Scopes + +
+
+ + + + + + +
+
+
+ Custom Scopes +
+ + (Defined on key creation) + +
+
+
+
+ + Custom Scopes... + +
+
+ + + + +
+
+
+ Scope Validation +
+
+
+
+ + Scope Validation + +
+
+ + + + + + + + + + +
+
+
+ Database +
+ + Each doc has permission + +
+
+
+
+ + Database... + +
+
+ + + + +
+
+
+ Roles Validation +
+
+
+
+ + Roles Validation + +
+
+ + + + + + +
+
+
+ Roles +
+
+
+
+ + Roles + +
+
+ + + + + + +
+
+
+ Wildcard +
+ * +
+
+
+
+ + Wildcard... + +
+
+ + + + + + +
+
+
+ Guset +
+ role:guest +
+
+
+
+ + Guset... + +
+
+ + + + + + +
+
+
+ Member +
+ role:member +
+
+
+
+ + Member... + +
+
+ + + + + + +
+
+
+ App +
+ role:app +
+
+
+
+ + App... + +
+
+ + + + +
+
+
+ User ID +
+ user:[ID] +
+
+
+
+ + User ID... + +
+
+ + + + + + +
+
+
+ Team ID +
+ team:[ID] +
+
+
+
+ + Team ID... + +
+
+ + + + + + +
+
+
+ Team ID + Role +
+ team:[ID]/[ROLE] +
+
+
+
+ + Team ID + Role... + +
+
+ + + + +
+
+
+ Member ID +
+ member:[ID] +
+
+
+
+ + Member ID... + +
+
+ + + + +
+
+
+ Endpoints +
+ + Each endpoint has 1 scope + +
+
+
+
+ + Endpoints... + +
+
+ + + + + + +
+
+
+ Scopes +
+
+
+
+ + Scopes + +
+
+ + + + + + +
+
+
+ public +
+
+
+
+ + public + +
+
+ + + + + + +
+
+
+ account +
+
+
+
+ + account + +
+
+ + + + + + +
+
+
+ files.read +
+
+
+
+ + files.read + +
+
+ + + + +
+
+
+ files.write +
+
+
+
+ + files.write + +
+
+ + + + +
+
+
+ + ... + +
+
+
+
+ + ... + +
+
From 8de831196a09808926526d33f87652aff209ca61 Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Sat, 16 Jan 2021 12:33:56 +0200 Subject: [PATCH 029/156] Fixed old diagram --- docs/specs/overview.drawio.svg | 793 ++++++++++++++++++--------------- 1 file changed, 439 insertions(+), 354 deletions(-) diff --git a/docs/specs/overview.drawio.svg b/docs/specs/overview.drawio.svg index f520c0c1c..db04e4c96 100644 --- a/docs/specs/overview.drawio.svg +++ b/docs/specs/overview.drawio.svg @@ -1,641 +1,726 @@ - + - - - + + + -
+
- Session Cookie + Web
- - Session Cookie + + Web - - - + + + -
-
-
- Email / Password -
-
-
-
- - Email / Password - -
-
- - - - - - -
-
-
- OAuth Provider -
-
-
-
- - OAuth Provider - -
-
- - - - - -
-
-
- Granted with -
-
-
-
- - Granted with - -
-
- - - - -
+
- Member + Flutter
- - Member + + Flutter - - + + + -
-
-
- Granted with -
-
-
-
- - Granted with - -
-
- - - - -
+
- App + iOS
- - App + + iOS - - - + + + -
-
-
- JWT -
-
-
-
- - JWT - -
-
- - - - - - -
-
-
- AP Key -
-
-
-
- - AP Key - -
-
- - - - - -
-
-
- Granted with -
-
-
-
- - Granted with - -
-
- - - - -
+
- Guest + Android
- - Guest + + Android - - - + + + -
+
- Public Scopes + Servers
- - Public Scopes + + Servers - - - + + + -
+
- Member Scopes + Appwrite
- - Member Scopes + + Appwrite - - - + + + + + + + -
+
- Custom Scopes -
- - (Defined on key creation) - + Loadbalancer
- - Custom Scopes... + + Loadbalancer - + + + -
+
-
- Scope Validation +
+ Console
- - Scope Validation + + Console - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + -
+
-
+
+ APIs +
+
+
+ + + APIs + + + + + + + +
+
+
+ Pub/Sub (Redis) +
+
+
+
+ + Pub/Sub (Redis) + +
+
+ + + + + + + + +
+
+
+ Cache (Redis) +
+
+
+
+ + Cache (Redis) + +
+
+ + + + +
+
+
+ Database (MariaDB) +
+
+
+
+ + Database (MariaDB) + +
+
+ + + + +
+
+
+ Users +
+
+
+
+ + Users + +
+
+ + + + +
+
+
+ Account +
+
+
+
+ + Account + +
+
+ + + + +
+
+
+ Teams +
+
+
+
+ + Teams + +
+
+ + + + +
+
+
Database -
- - Each doc has permission -
- - Database... + + Database
- + -
+
-
- Roles Validation +
+ Storage
- - Roles Validation + + Storage - - - + -
+
-
- Roles +
+ Localization
- - Roles + + Localization - - - + -
+
+
+
+ Avatars +
+
+
+ + + Avatars + + + + + + + +
+
+
+ Health +
+
+
+
+ + Health + +
+
+ + + + +
+
+
+ SSL Gateway +
+
+
+
+ + SSL Gateway + +
+
+ + + + + + + + +
- Wildcard -
- * + Deletes
- - Wildcard... + + Deletes
- - - + + + + + -
+
- Guset -
- role:guest + Security Layer
- - Guset... + + Security Layer - - - + + + + + + + -
+
- Member -
- role:member + Usage
- - Member... + + Usage - - - + + + + + -
+
- App -
- role:app + Audits
- - App... + + Audits - + + + + + -
+
- User ID -
- user:[ID] + Mails
- - User ID... + + Mails - - - + -
+
- Team ID -
- team:[ID] + SMTP
- - Team ID... + + SMTP - - - + + + + + + + -
+
- Team ID + Role -
- team:[ID]/[ROLE] + Tasks
- - Team ID + Role... + + Tasks - + + + + + -
+
- Member ID -
- member:[ID] + Webhooks
- - Member ID... + + Webhooks - + + + + + + + -
-
-
- Endpoints -
- - Each endpoint has 1 scope - -
-
-
-
- - Endpoints... - -
-
- - - - - - -
-
-
- Scopes -
-
-
-
- - Scopes - -
-
- - - - - - -
+
- public + Functions
- - public + + Functions - - - + -
+
- account + Docker
- - account + + Docker - - - + -
+
- files.read + StatsD (Telegraf)
- - files.read + + StatsD (Telegraf) - + -
+
- files.write + TimeSeries (InfluxDB)
- - files.write + + TimeSeries (InfluxDB) - + + + + + + + -
+
- - ... - + Certs
- - ... + + Certs + + + + + + + + + +
+
+
+ Scheduler +
+
+
+
+ + Scheduler + +
+
+ + + + + + +
+
+
+ Letsencrypt +
+
+
+
+ + Letsencrypt + +
+
+ + + + +
+
+
+ AntiVirus (ClamAV) +
+
+
+
+ + AntiVirus (ClamAV)
From 1da4fa8168e9295282d8e8f0265f923c153b2a23 Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Sat, 16 Jan 2021 17:10:20 +0200 Subject: [PATCH 030/156] Updated docs and logs --- .travis.yml | 4 +- docker-compose.yml | 2 +- docs/specs/authentication.drawio.svg | 499 +++++++++++++++++---------- 3 files changed, 321 insertions(+), 184 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6d87d519a..afb64c0c8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,9 +20,7 @@ before_install: - sudo service docker start - > if [ ! -z "${DOCKERHUB_PULL_USERNAME:-}" ]; then - set +x - echo "${DOCKERHUB_PULL_PASSWORD}" | docker login --username "${DOCKERHUB_PULL_USERNAME}" --password-stdin - set -x + echo "${DOCKERHUB_PULL_PASSWORD}" | docker login --username "${DOCKERHUB_PULL_USERNAME}" --password-stdin fi - docker --version diff --git a/docker-compose.yml b/docker-compose.yml index be55f7acb..38858e291 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -410,7 +410,7 @@ services: # Dev Tools Start ------------------------------------------------------------------------------------------ # - # The Appwrite Team uses the following tools to help debug, monitor and diagnose the Appwrite stack 🪛 + # The Appwrite Team uses the following tools to help debug, monitor and diagnose the Appwrite stack # # Here is a description of the different tools and why are we using them: # diff --git a/docs/specs/authentication.drawio.svg b/docs/specs/authentication.drawio.svg index 0f665a33e..0a34125ad 100644 --- a/docs/specs/authentication.drawio.svg +++ b/docs/specs/authentication.drawio.svg @@ -1,32 +1,32 @@ - + - - - + + + -
+
- Session Cookie + Secure Cookie
- - Session Cookie + + Secure Cookie - - - + + + -
+
Email / Password @@ -34,18 +34,18 @@
- + Email / Password - - - + + + -
+
OAuth Provider @@ -53,34 +53,18 @@
- + OAuth Provider - - + + + -
-
-
- Granted with -
-
-
-
- - Granted with - -
-
- - - - -
+
Member @@ -88,34 +72,18 @@
- + Member - - + + + -
-
-
- Granted with -
-
-
-
- - Granted with - -
-
- - - - -
+
App @@ -123,18 +91,18 @@
- + App - - - + + + -
+
JWT @@ -142,18 +110,18 @@
- + JWT - - - + + + -
+
AP Key @@ -161,17 +129,17 @@
- + AP Key - - + + -
+
Granted with @@ -179,16 +147,16 @@
- + Granted with - + -
+
Guest @@ -196,58 +164,58 @@
- + Guest - - - + + + -
+
-
+
Public Scopes
- + Public Scopes - - - + + + -
+
-
+
Member Scopes
- + Member Scopes - - - + + + -
+
-
+
Custom Scopes
@@ -257,16 +225,16 @@
- + Custom Scopes... - + -
+
Scope Validation @@ -274,22 +242,22 @@
- + Scope Validation - - - - - - - + + + + + + + -
+
Database @@ -301,16 +269,16 @@
- + Database... - + -
+
Roles Validation @@ -318,18 +286,18 @@
- + Roles Validation - - - + + + -
+
Roles @@ -337,18 +305,18 @@
- + Roles - - - + + + -
+
Wildcard @@ -358,18 +326,18 @@
- + Wildcard... - - - + + + -
+
Guset @@ -379,18 +347,18 @@
- + Guset... - - - + + + -
+
Member @@ -400,18 +368,18 @@
- + Member... - - - + + + -
+
App @@ -421,16 +389,16 @@
- + App... - + -
+
User ID @@ -440,18 +408,18 @@
- + User ID... - - - + + + -
+
Team ID @@ -461,18 +429,18 @@
- + Team ID... - - - + + + -
+
Team ID + Role @@ -482,16 +450,16 @@
- + Team ID + Role... - + -
+
Member ID @@ -501,16 +469,16 @@
- + Member ID... - + -
+
Endpoints @@ -522,18 +490,18 @@
- + Endpoints... - - - + + + -
+
Scopes @@ -541,18 +509,18 @@
- + Scopes - - - + + + -
+
public @@ -560,18 +528,18 @@
- + public - - - + + + -
+
account @@ -579,18 +547,18 @@
- + account - - - + + + -
+
files.read @@ -598,16 +566,16 @@
- + files.read - + -
+
files.write @@ -615,16 +583,16 @@
- + files.write - + -
+
@@ -634,11 +602,182 @@
- + ... + + + + + + +
+
+
+ Guest Role +
+ (only) +
+
+
+
+ + Guest Role... + +
+
+ + + + +
+
+
+ Member / User / Team Roles +
+
+
+
+ + Member / User / Team... + +
+
+ + + + + +
+
+
+ Granted with +
+
+
+
+ + Granted with + +
+
+ + + + +
+
+
+ No Role Base +
+ Authentication +
+
+
+
+ + No Role Base... + +
+
+ + + + + +
+
+
+ Granted with +
+
+
+
+ + Granted with + +
+
+ + + + + + +
+
+
+ Team Invite +
+
+
+
+ + Team Invite + +
+
+ + + + + + +
+
+
+ HTTP Header +
+ X-Appwrite-Key +
+
+
+
+ + HTTP Header... + +
+
+ + + + +
+
+
+ HTTP Header +
+ X-Appwrite-JWT +
+
+
+
+ + HTTP Header... + +
+
+ + + + +
+
+
+ + Not Released Yet + +
+
+
+
+ + Not Released Yet + +
+
From 47f5e871baa07f2833db79ce103ece5075247313 Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Sun, 17 Jan 2021 01:38:13 +0200 Subject: [PATCH 031/156] Added scheduling --- app/controllers/api/functions.php | 82 ++++++++++++++++++++++++------- app/workers/functions.php | 49 ++++++++++++++---- 2 files changed, 103 insertions(+), 28 deletions(-) diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 0572b8137..7a9996fbd 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -44,6 +44,9 @@ App::post('/v1/functions') ->inject('response') ->inject('projectDB') ->action(function ($name, $execute, $env, $vars, $events, $schedule, $timeout, $response, $projectDB) { + /** @var Appwrite\Utopia\Response $response */ + /** @var Appwrite\Database\Database $projectDB */ + $function = $projectDB->createDocument([ '$collection' => Database::SYSTEM_COLLECTION_FUNCTIONS, '$permissions' => [ @@ -91,6 +94,9 @@ App::get('/v1/functions') ->inject('response') ->inject('projectDB') ->action(function ($search, $limit, $offset, $orderType, $response, $projectDB) { + /** @var Appwrite\Utopia\Response $response */ + /** @var Appwrite\Database\Database $projectDB */ + $results = $projectDB->getCollection([ 'limit' => $limit, 'offset' => $offset, @@ -122,6 +128,9 @@ App::get('/v1/functions/:functionId') ->inject('response') ->inject('projectDB') ->action(function ($functionId, $response, $projectDB) { + /** @var Appwrite\Utopia\Response $response */ + /** @var Appwrite\Database\Database $projectDB */ + $function = $projectDB->getDocument($functionId); if (empty($function->getId()) || Database::SYSTEM_COLLECTION_FUNCTIONS != $function->getCollection()) { @@ -272,7 +281,12 @@ App::put('/v1/functions/:functionId') ->param('timeout', 15, new Range(1, 900), 'Function maximum execution time in seconds.', true) ->inject('response') ->inject('projectDB') - ->action(function ($functionId, $name, $execute, $vars, $events, $schedule, $timeout, $response, $projectDB) { + ->inject('project') + ->action(function ($functionId, $name, $execute, $vars, $events, $schedule, $timeout, $response, $projectDB, $project) { + /** @var Appwrite\Utopia\Response $response */ + /** @var Appwrite\Database\Database $projectDB */ + /** @var Appwrite\Database\Document $project */ + $function = $projectDB->getDocument($functionId); if (empty($function->getId()) || Database::SYSTEM_COLLECTION_FUNCTIONS != $function->getCollection()) { @@ -291,28 +305,23 @@ App::put('/v1/functions/:functionId') 'vars' => $vars, 'events' => $events, 'schedule' => $schedule, - 'schedulePrevious' => null, 'scheduleNext' => $next, - 'timeout' => $timeout, + 'timeout' => $timeout, ])); - if ($next) { - ResqueScheduler::enqueueAt($next, 'v1-functions', 'FunctionsV1', [ - - ]); - - // ->setParam('projectId', $project->getId()) - // ->setParam('event', $route->getLabel('event', '')) - // ->setParam('payload', []) - // ->setParam('functionId', null) - // ->setParam('executionId', null) - // ->setParam('trigger', 'event') - } - if (false === $function) { throw new Exception('Failed saving function to DB', 500); } + if ($next) { + ResqueScheduler::enqueueAt($next, 'v1-functions', 'FunctionsV1', [ + 'projectId' => $project->getId(), + 'functionId' => $function->getId(), + 'executionId' => null, + 'trigger' => 'schedule', + ]); // Async task rescheduale + } + $response->dynamic($function, Response::MODEL_FUNCTION); }); @@ -331,7 +340,12 @@ App::patch('/v1/functions/:functionId/tag') ->param('tag', '', new UID(), 'Tag unique ID.') ->inject('response') ->inject('projectDB') - ->action(function ($functionId, $tag, $response, $projectDB) { + ->inject('project') + ->action(function ($functionId, $tag, $response, $projectDB, $project) { + /** @var Appwrite\Utopia\Response $response */ + /** @var Appwrite\Database\Database $projectDB */ + /** @var Appwrite\Database\Document $project */ + $function = $projectDB->getDocument($functionId); $tag = $projectDB->getDocument($tag); @@ -344,14 +358,23 @@ App::patch('/v1/functions/:functionId/tag') } $schedule = $function->getAttribute('schedule', ''); - $cron = (!empty($function->getAttribute('tag')&& !empty($schedule))) ? CronExpression::factory($schedule) : null; - $next = (!empty($function->getAttribute('tag')&& !empty($schedule))) ? $cron->getNextRunDate()->format('U') : null; + $cron = (empty($function->getAttribute('tag') && !empty($schedule))) ? CronExpression::factory($schedule) : null; + $next = (empty($function->getAttribute('tag') && !empty($schedule))) ? $cron->getNextRunDate()->format('U') : null; $function = $projectDB->updateDocument(array_merge($function->getArrayCopy(), [ 'tag' => $tag->getId(), 'scheduleNext' => $next, ])); + if ($next) { // Init first schedule + ResqueScheduler::enqueueAt($next, 'v1-functions', 'FunctionsV1', [ + 'projectId' => $project->getId(), + 'functionId' => $function->getId(), + 'executionId' => null, + 'trigger' => 'schedule', + ]); // Async task rescheduale + } + if (false === $function) { throw new Exception('Failed saving function to DB', 500); } @@ -418,6 +441,11 @@ App::post('/v1/functions/:functionId/tags') ->inject('projectDB') ->inject('usage') ->action(function ($functionId, $command, $code, $request, $response, $projectDB, $usage) { + /** @var Utopia\Swoole\Request $request */ + /** @var Appwrite\Utopia\Response $response */ + /** @var Appwrite\Database\Database $projectDB */ + /** @var Appwrite\Event\Event $usage */ + $function = $projectDB->getDocument($functionId); if (empty($function->getId()) || Database::SYSTEM_COLLECTION_FUNCTIONS != $function->getCollection()) { @@ -506,6 +534,9 @@ App::get('/v1/functions/:functionId/tags') ->inject('response') ->inject('projectDB') ->action(function ($functionId, $search, $limit, $offset, $orderType, $response, $projectDB) { + /** @var Appwrite\Utopia\Response $response */ + /** @var Appwrite\Database\Database $projectDB */ + $function = $projectDB->getDocument($functionId); if (empty($function->getId()) || Database::SYSTEM_COLLECTION_FUNCTIONS != $function->getCollection()) { @@ -545,6 +576,9 @@ App::get('/v1/functions/:functionId/tags/:tagId') ->inject('response') ->inject('projectDB') ->action(function ($functionId, $tagId, $response, $projectDB) { + /** @var Appwrite\Utopia\Response $response */ + /** @var Appwrite\Database\Database $projectDB */ + $function = $projectDB->getDocument($functionId); if (empty($function->getId()) || Database::SYSTEM_COLLECTION_FUNCTIONS != $function->getCollection()) { @@ -581,6 +615,10 @@ App::delete('/v1/functions/:functionId/tags/:tagId') ->inject('projectDB') ->inject('usage') ->action(function ($functionId, $tagId, $response, $projectDB, $usage) { + /** @var Appwrite\Utopia\Response $response */ + /** @var Appwrite\Database\Database $projectDB */ + /** @var Appwrite\Event\Event $usage */ + $function = $projectDB->getDocument($functionId); if (empty($function->getId()) || Database::SYSTEM_COLLECTION_FUNCTIONS != $function->getCollection()) { @@ -727,6 +765,9 @@ App::get('/v1/functions/:functionId/executions') ->inject('response') ->inject('projectDB') ->action(function ($functionId, $search, $limit, $offset, $orderType, $response, $projectDB) { + /** @var Appwrite\Utopia\Response $response */ + /** @var Appwrite\Database\Database $projectDB */ + $function = $projectDB->getDocument($functionId); if (empty($function->getId()) || Database::SYSTEM_COLLECTION_FUNCTIONS != $function->getCollection()) { @@ -766,6 +807,9 @@ App::get('/v1/functions/:functionId/executions/:executionId') ->inject('response') ->inject('projectDB') ->action(function ($functionId, $executionId, $response, $projectDB) { + /** @var Appwrite\Utopia\Response $response */ + /** @var Appwrite\Database\Database $projectDB */ + $function = $projectDB->getDocument($functionId); if (empty($function->getId()) || Database::SYSTEM_COLLECTION_FUNCTIONS != $function->getCollection()) { diff --git a/app/workers/functions.php b/app/workers/functions.php index 2712dc74f..222b76a60 100644 --- a/app/workers/functions.php +++ b/app/workers/functions.php @@ -6,6 +6,7 @@ use Appwrite\Database\Adapter\MySQL as MySQLAdapter; use Appwrite\Database\Adapter\Redis as RedisAdapter; use Appwrite\Database\Validator\Authorization; use Appwrite\Event\Event; +use Cron\CronExpression; use Swoole\Runtime; use Utopia\App; use Utopia\CLI\Console; @@ -27,7 +28,7 @@ $environments = Config::getParam('environments'); $warmupStart = \microtime(true); Co\run(function() use ($environments) { // Warmup: make sure images are ready to run fast 🚀 - Swoole\Runtime::enableCoroutine(SWOOLE_HOOK_ALL); + Runtime::enableCoroutine(SWOOLE_HOOK_ALL); foreach($environments as $environment) { go(function() use ($environment) { @@ -79,14 +80,6 @@ $stdout = \explode("\n", $stdout); \parse_str($value, $container); if(isset($container['name'])) { - // $labels = []; - // $temp = explode(',', $container['labels'] ?? []); - - // foreach($temp as &$label) { - // $label = explode('=', $label); - // $labels[$label[0] || 0] = $label[1] || ''; - // } - $container = [ 'name' => $container['name'], 'online' => (\substr($container['status'], 0, 2) === 'Up'), @@ -142,6 +135,7 @@ class FunctionsV1 $executionId = $this->args['executionId'] ?? ''; $trigger = $this->args['trigger'] ?? ''; $event = $this->args['event'] ?? ''; + $scheduleOriginal = $this->args['scheduleOriginal'] ?? ''; $payload = (!empty($this->args['payload'])) ? json_encode($this->args['payload']) : ''; $database = new Database(); @@ -210,6 +204,43 @@ class FunctionsV1 * On failure add error count * If error count bigger than allowed change status to pause */ + + // Reschedule + Authorization::disable(); + $function = $database->getDocument($functionId); + Authorization::reset(); + + if (empty($function->getId()) || Database::SYSTEM_COLLECTION_FUNCTIONS != $function->getCollection()) { + throw new Exception('Function not found ('.$functionId.')'); + } + + if($scheduleOriginal && $scheduleOriginal !== $function->getAttribute('schedule')) { // Schedule has changed from previous run, ignore this run. + return; + } + + $cron = CronExpression::factory($function->getAttribute('schedule')); + $next = (int) $cron->getNextRunDate()->format('U'); + + $function + ->setAttribute('scheduleNext', $next) + ->setAttribute('schedulePrevious', \time()) + ; + + $function = $database->updateDocument(array_merge($function->getArrayCopy(), [ + 'scheduleNext' => $next, + ])); + + ResqueScheduler::enqueueAt($next, 'v1-functions', 'FunctionsV1', [ + 'projectId' => $projectId, + 'functionId' => $function->getId(), + 'executionId' => null, + 'trigger' => 'schedule', + 'scheduleOriginal' => $function->getAttribute('schedule', ''), + ]); // Async task rescheduale + + Swoole\Coroutine\run(function () use ($trigger, $projectId, $executionId, $database, $function) { + $this->execute($trigger, $projectId, $executionId, $database, $function); + }); break; From f16113dbb3a6dc1b99d3a1566283b81042f2e401 Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Sun, 17 Jan 2021 02:07:43 +0200 Subject: [PATCH 032/156] Some minor fixes --- app/controllers/api/functions.php | 8 +++++--- app/workers/functions.php | 4 ++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 7a9996fbd..f8f70796c 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -293,6 +293,7 @@ App::put('/v1/functions/:functionId') throw new Exception('Function not found', 404); } + $original = $function->getAttribute('schedule', ''); $cron = (!empty($function->getAttribute('tag', null)) && !empty($schedule)) ? CronExpression::factory($schedule) : null; $next = (!empty($function->getAttribute('tag', null)) && !empty($schedule)) ? $cron->getNextRunDate()->format('U') : null; @@ -313,7 +314,7 @@ App::put('/v1/functions/:functionId') throw new Exception('Failed saving function to DB', 500); } - if ($next) { + if ($next && $schedule !== $original) { ResqueScheduler::enqueueAt($next, 'v1-functions', 'FunctionsV1', [ 'projectId' => $project->getId(), 'functionId' => $function->getId(), @@ -358,8 +359,9 @@ App::patch('/v1/functions/:functionId/tag') } $schedule = $function->getAttribute('schedule', ''); - $cron = (empty($function->getAttribute('tag') && !empty($schedule))) ? CronExpression::factory($schedule) : null; - $next = (empty($function->getAttribute('tag') && !empty($schedule))) ? $cron->getNextRunDate()->format('U') : null; + var_dump($schedule); + $cron = (empty($function->getAttribute('tag')) && !empty($schedule)) ? CronExpression::factory($schedule) : null; + $next = (empty($function->getAttribute('tag')) && !empty($schedule)) ? $cron->getNextRunDate()->format('U') : null; $function = $projectDB->updateDocument(array_merge($function->getArrayCopy(), [ 'tag' => $tag->getId(), diff --git a/app/workers/functions.php b/app/workers/functions.php index 222b76a60..6c3ba252b 100644 --- a/app/workers/functions.php +++ b/app/workers/functions.php @@ -226,10 +226,14 @@ class FunctionsV1 ->setAttribute('schedulePrevious', \time()) ; + Authorization::disable(); + $function = $database->updateDocument(array_merge($function->getArrayCopy(), [ 'scheduleNext' => $next, ])); + Authorization::reset(); + ResqueScheduler::enqueueAt($next, 'v1-functions', 'FunctionsV1', [ 'projectId' => $projectId, 'functionId' => $function->getId(), From 26faaae5fce52ea74d269cdc0352b8a3a8e70d6e Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Sun, 17 Jan 2021 02:08:16 +0200 Subject: [PATCH 033/156] Removed log --- app/controllers/api/functions.php | 1 - 1 file changed, 1 deletion(-) diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index f8f70796c..56bf9bfb2 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -359,7 +359,6 @@ App::patch('/v1/functions/:functionId/tag') } $schedule = $function->getAttribute('schedule', ''); - var_dump($schedule); $cron = (empty($function->getAttribute('tag')) && !empty($schedule)) ? CronExpression::factory($schedule) : null; $next = (empty($function->getAttribute('tag')) && !empty($schedule)) ? $cron->getNextRunDate()->format('U') : null; From b719a030e0edd68cd5f5c4dfaa28d0b67044df4a Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Sun, 17 Jan 2021 08:17:19 +0200 Subject: [PATCH 034/156] removed db instance, added time log --- app/tasks/maintenance.php | 23 ++++++----------------- 1 file changed, 6 insertions(+), 17 deletions(-) diff --git a/app/tasks/maintenance.php b/app/tasks/maintenance.php index f8786bcac..78930821c 100644 --- a/app/tasks/maintenance.php +++ b/app/tasks/maintenance.php @@ -4,23 +4,13 @@ global $cli; require_once __DIR__.'/../init.php'; -use Appwrite\Database\Database; -use Appwrite\Database\Adapter\MySQL as MySQLAdapter; -use Appwrite\Database\Adapter\Redis as RedisAdapter; use Appwrite\Event\Event; use Utopia\App; use Utopia\CLI\Console; -use Utopia\Config\Config; -// TODO: Think of a better way to access consoleDB -function getConsoleDB() { - global $register; - $consoleDB = new Database(); - $consoleDB->setAdapter(new RedisAdapter(new MySQLAdapter($register), $register)); - $consoleDB->setNamespace('app_console'); // Main DB - $consoleDB->setMocks(Config::getParam('collections', [])); - return $consoleDB; -} +Console::title('Maintenance V1'); + +Console::success(APP_NAME.' maintenance process v1 has started'); function notifyDeleteExecutionLogs() { @@ -54,10 +44,9 @@ $cli //Convert Seconds to microseconds $intervalMicroseconds = $interval * 1000000; - $consoleDB = getConsoleDB(); - - Console::loop(function() use ($consoleDB, $interval){ - Console::info("[ MAINTENANCE TASK ] Notifying deletes workers every {$interval} seconds"); + Console::loop(function() use ($interval){ + $time = date('d-m-Y H:i:s', time()); + Console::info("[{$time}] Notifying deletes workers every {$interval} seconds"); notifyDeleteExecutionLogs(); notifyDeleteAbuseLogs($interval); notifyDeleteAuditLogs($interval); From c3971a68de7628825224b73206aa00c7b67beaf7 Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Sun, 17 Jan 2021 08:17:54 +0200 Subject: [PATCH 035/156] Removed DB vars, added dev mounts --- docker-compose.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 38858e291..e9d82cbf0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -319,6 +319,9 @@ services: context: . networks: - appwrite + volumes: + - ./app:/usr/src/code/app + - ./src:/usr/src/code/src depends_on: - redis environment: @@ -326,11 +329,6 @@ services: - _APP_REDIS_HOST - _APP_REDIS_PORT - _APP_MAINTENANCE_INTERVAL - - _APP_DB_HOST - - _APP_DB_PORT - - _APP_DB_SCHEMA - - _APP_DB_USER - - _APP_DB_PASS appwrite-schedule: entrypoint: schedule From 5aff018d116ed6ffd9711ccf700616741ac50af3 Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Sun, 17 Jan 2021 15:49:11 +0200 Subject: [PATCH 036/156] Fix for error reporting when no route found --- app/controllers/general.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/general.php b/app/controllers/general.php index dfb28d50a..94ef9f5c7 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -254,7 +254,7 @@ App::error(function ($error, $utopia, $request, $response, $layout, $project) { $route = $utopia->match($request); $template = ($route) ? $route->getLabel('error', null) : null; - if (php_sapi_name() === 'cli') { + if (php_sapi_name() === 'cli' && $route) { Console::error('[Error] Method: '.$route->getMethod()); Console::error('[Error] URL: '.$route->getURL()); Console::error('[Error] Type: '.get_class($error)); From bf2c8b4010ff6575c4560151e5381b2531b8dbe1 Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Sun, 17 Jan 2021 15:50:01 +0200 Subject: [PATCH 037/156] Fixed error reporting --- app/controllers/general.php | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/app/controllers/general.php b/app/controllers/general.php index 94ef9f5c7..0ac19d589 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -254,9 +254,12 @@ App::error(function ($error, $utopia, $request, $response, $layout, $project) { $route = $utopia->match($request); $template = ($route) ? $route->getLabel('error', null) : null; - if (php_sapi_name() === 'cli' && $route) { - Console::error('[Error] Method: '.$route->getMethod()); - Console::error('[Error] URL: '.$route->getURL()); + if (php_sapi_name() === 'cli') { + if($route) { + Console::error('[Error] Method: '.$route->getMethod()); + Console::error('[Error] URL: '.$route->getURL()); + } + Console::error('[Error] Type: '.get_class($error)); Console::error('[Error] Message: '.$error->getMessage()); Console::error('[Error] File: '.$error->getFile()); From a8b666dfc31accea034f55dfa0da5c242293a8ef Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Mon, 18 Jan 2021 08:29:26 +0200 Subject: [PATCH 038/156] First commit --- app/controllers/general.php | 4 + app/views/console/account/index.phtml | 16 ++ app/views/console/comps/footer.phtml | 3 + app/views/console/comps/header.phtml | 13 ++ app/views/console/database/collection.phtml | 7 + app/views/console/database/document.phtml | 4 + app/views/console/database/index.phtml | 2 + app/views/console/functions/function.phtml | 14 ++ app/views/console/functions/index.phtml | 2 + app/views/console/home/index.phtml | 13 ++ app/views/console/index.phtml | 7 +- app/views/console/keys/index.phtml | 6 + app/views/console/settings/index.phtml | 16 ++ app/views/console/storage/index.phtml | 6 + app/views/console/tasks/index.phtml | 6 + app/views/console/users/index.phtml | 6 + app/views/console/users/team.phtml | 8 + app/views/console/users/user.phtml | 10 + app/views/console/webhooks/index.phtml | 6 + app/views/home/auth/join.phtml | 2 + app/views/home/auth/recovery.phtml | 2 + app/views/home/auth/recovery/reset.phtml | 2 + app/views/home/auth/signin.phtml | 2 + app/views/home/auth/signup.phtml | 2 + gulpfile.js | 2 + public/dist/scripts/app-all.js | 18 +- public/dist/scripts/app.js | 18 +- public/scripts/services/api.js | 199 ++++++++++++++++++++ public/scripts/views/analytics/activity.js | 17 ++ public/scripts/views/analytics/event.js | 2 +- 30 files changed, 407 insertions(+), 8 deletions(-) create mode 100644 public/scripts/services/api.js create mode 100644 public/scripts/views/analytics/activity.js diff --git a/app/controllers/general.php b/app/controllers/general.php index 0ac19d589..5cc69346f 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -233,6 +233,10 @@ App::options(function ($request, $response) { $origin = $request->getOrigin(); + var_dump('-----------'); + var_dump($origin); + var_dump('-----------'); + $response ->addHeader('Server', 'Appwrite') ->addHeader('Access-Control-Allow-Methods', 'GET, POST, PUT, PATCH, DELETE') diff --git a/app/views/console/account/index.phtml b/app/views/console/account/index.phtml index 84a18521d..3f1e7decc 100644 --- a/app/views/console/account/index.phtml +++ b/app/views/console/account/index.phtml @@ -29,6 +29,8 @@
Update Password PLEASE NOTE: Account deletion is irreversible.

getParam('version', '').'.'.APP_CACHE_BUSTER;