From 878021b9a31d69683ad65a7b61d196d9a48b0758 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Thu, 17 Nov 2022 18:20:17 +0530 Subject: [PATCH 1/3] feat: disable auth on console project --- app/controllers/api/storage.php | 1 + app/init.php | 9 +++--- app/workers/builds.php | 2 +- app/workers/messaging.php | 2 +- composer.json | 1 + composer.lock | 49 ++++++++++++++++++++++++++++++++- src/Appwrite/Resque/Worker.php | 2 +- 7 files changed, 58 insertions(+), 8 deletions(-) diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index e2a3c2de6..1aa4b3bf9 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -41,6 +41,7 @@ use Utopia\Validator\HexColor; use Utopia\Validator\Range; use Utopia\Validator\Text; use Utopia\Validator\WhiteList; +use Utopia\DSN\DSN; use Utopia\Swoole\Request; App::post('/v1/storage/buckets') diff --git a/app/init.php b/app/init.php index 8768103e8..5846a7ba8 100644 --- a/app/init.php +++ b/app/init.php @@ -54,7 +54,6 @@ use Utopia\Config\Config; use Utopia\Locale\Locale; use Utopia\Registry\Registry; use Utopia\Storage\Device; -use Utopia\Storage\Storage; use Utopia\DSN\DSN; use Utopia\Storage\Device\Backblaze; use Utopia\Storage\Device\DOSpaces; @@ -75,6 +74,7 @@ use Appwrite\Event\Func; use MaxMind\Db\Reader; use PHPMailer\PHPMailer\PHPMailer; use Swoole\Database\PDOProxy; +use Utopia\CLI\Console; use Utopia\Queue; const APP_NAME = 'Appwrite'; @@ -593,7 +593,7 @@ $register->set('pools', function () { $dsnUser = $dsn->getUser(); $dsnPass = $dsn->getPassword(); $dsnScheme = $dsn->getScheme(); - $dsnDatabase = $dsn->getDatabase(); + $dsnDatabase = $dsn->getPath(); if (!in_array($dsnScheme, $schemes)) { throw new Exception(Exception::GENERAL_SERVER_ERROR, "Invalid console database scheme"); @@ -653,7 +653,7 @@ $register->set('pools', function () { default => null }; - $adapter->setDefaultDatabase($dsn->getDatabase()); + $adapter->setDefaultDatabase($dsn->getPath()); break; case 'pubsub': $adapter = $resource(); @@ -1035,6 +1035,7 @@ App::setResource('console', function () { 'legalAddress' => '', 'legalTaxId' => '', 'auths' => [ + 'invites' => false, 'limit' => (App::getEnv('_APP_CONSOLE_WHITELIST_ROOT', 'enabled') === 'enabled') ? 1 : 0, // limit signup to 1 user ], 'authWhitelistEmails' => (!empty(App::getEnv('_APP_CONSOLE_WHITELIST_EMAILS', null))) ? \explode(',', App::getEnv('_APP_CONSOLE_WHITELIST_EMAILS', null)) : [], @@ -1123,7 +1124,7 @@ function getDevice($root): Device $bucket = $dsn->getPath(); $region = $dsn->getParam('region'); } catch (\Exception $e) { - Console::eor($e->getMessage() . 'Invalid DSN. Defaulting to Local storage.'); + Console::error($e->getMessage() . 'Invalid DSN. Defaulting to Local storage.'); $device = STORAGE_DEVICE_LOCAL; } diff --git a/app/workers/builds.php b/app/workers/builds.php index 348794c74..6535a56b0 100644 --- a/app/workers/builds.php +++ b/app/workers/builds.php @@ -11,7 +11,7 @@ use Utopia\Database\DateTime; use Utopia\App; use Utopia\CLI\Console; use Utopia\Database\ID; -use Utopia\Storage\Storage; +use Utopia\DSN\DSN; use Utopia\Database\Document; use Utopia\Config\Config; use Utopia\Database\Validator\Authorization; diff --git a/app/workers/messaging.php b/app/workers/messaging.php index 8d13919f2..4c82a1abf 100644 --- a/app/workers/messaging.php +++ b/app/workers/messaging.php @@ -6,7 +6,7 @@ use Appwrite\SMS\Adapter\TextMagic; use Appwrite\SMS\Adapter\Twilio; use Appwrite\SMS\Adapter\Msg91; use Appwrite\SMS\Adapter\Vonage; -use Appwrite\DSN\DSN; +use Utopia\DSN\DSN; use Appwrite\Resque\Worker; use Appwrite\SMS\Adapter; use Utopia\App; diff --git a/composer.json b/composer.json index ffd9c6b6b..2e6098c4c 100644 --- a/composer.json +++ b/composer.json @@ -54,6 +54,7 @@ "utopia-php/framework": "0.25.*", "utopia-php/image": "0.5.*", "utopia-php/queue": "0.4.*", + "utopia-php/dsn": "0.1.*", "utopia-php/locale": "0.4.*", "utopia-php/logger": "0.3.*", "utopia-php/orchestration": "0.9.*", diff --git a/composer.lock b/composer.lock index 1862ac982..54caaa313 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "a673091aa6bd8ef01380b63245427c93", + "content-hash": "9c2365876e32662c9c62e3a4c738fafe", "packages": [ { "name": "adhocore/jwt", @@ -1943,6 +1943,53 @@ }, "time": "2020-02-23T07:40:02+00:00" }, + { + "name": "utopia-php/dsn", + "version": "0.1.0", + "source": { + "type": "git", + "url": "https://github.com/utopia-php/dsn.git", + "reference": "17a5935eab1b89fb4b95600db50a1b6d5faa6cea" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/utopia-php/dsn/zipball/17a5935eab1b89fb4b95600db50a1b6d5faa6cea", + "reference": "17a5935eab1b89fb4b95600db50a1b6d5faa6cea", + "shasum": "" + }, + "require": { + "php": ">=8.0" + }, + "require-dev": { + "laravel/pint": "1.2.*", + "phpunit/phpunit": "^9.3", + "squizlabs/php_codesniffer": "^3.6", + "vimeo/psalm": "4.0.1" + }, + "type": "library", + "autoload": { + "psr-4": { + "Utopia\\DSN\\": "src/DSN" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "description": "A simple library for parsing and managing Data Source Names ( DSNs )", + "keywords": [ + "dsn", + "framework", + "php", + "upf", + "utopia" + ], + "support": { + "issues": "https://github.com/utopia-php/dsn/issues", + "source": "https://github.com/utopia-php/dsn/tree/0.1.0" + }, + "time": "2022-10-26T10:06:20+00:00" + }, { "name": "utopia-php/framework", "version": "0.25.0", diff --git a/src/Appwrite/Resque/Worker.php b/src/Appwrite/Resque/Worker.php index 8503e18c3..5bf3e254e 100644 --- a/src/Appwrite/Resque/Worker.php +++ b/src/Appwrite/Resque/Worker.php @@ -7,9 +7,9 @@ use Utopia\App; use Utopia\Cache\Cache; use Utopia\Config\Config; use Utopia\Cache\Adapter\Sharding; +use Utopia\CLI\Console; use Utopia\Database\Database; use Utopia\Storage\Device; -use Utopia\Storage\Storage; use Utopia\Storage\Device\Local; use Utopia\Storage\Device\DOSpaces; use Utopia\Storage\Device\Linode; From c4a5c737179855b57f117d8fb9bcee2d0e583a76 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Thu, 17 Nov 2022 23:13:59 +0530 Subject: [PATCH 2/3] feat: use new storage library --- app/controllers/api/storage.php | 11 +---------- app/init.php | 24 +++++++++--------------- app/workers/builds.php | 6 +++--- composer.json | 2 +- composer.lock | 14 +++++++------- src/Appwrite/Resque/Worker.php | 17 +++++++++-------- 6 files changed, 30 insertions(+), 44 deletions(-) diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index 1aa4b3bf9..3141e62c4 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -495,16 +495,7 @@ App::post('/v1/storage/buckets/:bucketId/files') } if ($chunksUploaded === $chunks) { - $connection = App::getEnv('_APP_CONNECTIONS_STORAGE', ''); /** @TODO : move this to the registry or someplace else */ - $device = STORAGE_DEVICE_LOCAL; - try { - $dsn = new DSN($connection); - $device = $dsn->getScheme(); - } catch (\Exception $e) { - $device = STORAGE_DEVICE_LOCAL; - } - - if (App::getEnv('_APP_STORAGE_ANTIVIRUS') === 'enabled' && $bucket->getAttribute('antivirus', true) && $fileSize <= APP_LIMIT_ANTIVIRUS && $device === STORAGE_DEVICE_LOCAL) { + if (App::getEnv('_APP_STORAGE_ANTIVIRUS') === 'enabled' && $bucket->getAttribute('antivirus', true) && $fileSize <= APP_LIMIT_ANTIVIRUS && $deviceFiles->getType() === Storage::DEVICE_LOCAL) { $antivirus = new Network( App::getEnv('_APP_STORAGE_ANTIVIRUS_HOST', 'clamav'), (int) App::getEnv('_APP_STORAGE_ANTIVIRUS_PORT', 3310) diff --git a/app/init.php b/app/init.php index 5846a7ba8..323d931ad 100644 --- a/app/init.php +++ b/app/init.php @@ -76,6 +76,7 @@ use PHPMailer\PHPMailer\PHPMailer; use Swoole\Database\PDOProxy; use Utopia\CLI\Console; use Utopia\Queue; +use Utopia\Storage\Storage; const APP_NAME = 'Appwrite'; const APP_DOMAIN = 'appwrite.io'; @@ -160,13 +161,6 @@ const DELETE_TYPE_SCHEDULES = 'schedules'; const COMPRESSION_TYPE_NONE = 'none'; const COMPRESSION_TYPE_GZIP = 'gzip'; const COMPRESSION_TYPE_ZSTD = 'zstd'; -// Storage Device Types -const STORAGE_DEVICE_LOCAL = 'file'; -const STORAGE_DEVICE_S3 = 's3'; -const STORAGE_DEVICE_DO_SPACES = 'dospaces'; -const STORAGE_DEVICE_BACKBLAZE = 'backblaze'; -const STORAGE_DEVICE_LINODE = 'linode'; -const STORAGE_DEVICE_WASABI = 'wasabi'; // Mail Types const MAIL_TYPE_VERIFICATION = 'verification'; const MAIL_TYPE_MAGIC_SESSION = 'magicSession'; @@ -1110,7 +1104,7 @@ function getDevice($root): Device $connection = App::getEnv('_APP_CONNECTIONS_STORAGE', ''); $acl = 'private'; - $device = STORAGE_DEVICE_LOCAL; + $device = Storage::DEVICE_LOCAL; $accessKey = ''; $accessSecret = ''; $bucket = ''; @@ -1125,21 +1119,21 @@ function getDevice($root): Device $region = $dsn->getParam('region'); } catch (\Exception $e) { Console::error($e->getMessage() . 'Invalid DSN. Defaulting to Local storage.'); - $device = STORAGE_DEVICE_LOCAL; + $device = Storage::DEVICE_LOCAL; } switch ($device) { - case STORAGE_DEVICE_S3: + case Storage::DEVICE_S3: return new S3($root, $accessKey, $accessSecret, $bucket, $region, $acl); - case STORAGE_DEVICE_DO_SPACES: + case STORAGE::DEVICE_DO_SPACES: return new DOSpaces($root, $accessKey, $accessSecret, $bucket, $region, $acl); - case STORAGE_DEVICE_BACKBLAZE: + case Storage::DEVICE_BACKBLAZE: return new Backblaze($root, $accessKey, $accessSecret, $bucket, $region, $acl); - case STORAGE_DEVICE_LINODE: + case Storage::DEVICE_LINODE: return new Linode($root, $accessKey, $accessSecret, $bucket, $region, $acl); - case STORAGE_DEVICE_WASABI: + case Storage::DEVICE_WASABI: return new Wasabi($root, $accessKey, $accessSecret, $bucket, $region, $acl); - case STORAGE_DEVICE_LOCAL: + case Storage::DEVICE_LOCAL: default: return new Local($root); } diff --git a/app/workers/builds.php b/app/workers/builds.php index 6535a56b0..864987de7 100644 --- a/app/workers/builds.php +++ b/app/workers/builds.php @@ -15,6 +15,7 @@ use Utopia\DSN\DSN; use Utopia\Database\Document; use Utopia\Config\Config; use Utopia\Database\Validator\Authorization; +use Utopia\Storage\Storage; require_once __DIR__ . '/../init.php'; @@ -80,12 +81,11 @@ class BuildsV1 extends Worker } $connection = App::getEnv('_APP_CONNECTIONS_STORAGE', ''); /** @TODO : move this to the registry or someplace else */ - $device = STORAGE_DEVICE_LOCAL; + $device = Storage::DEVICE_LOCAL; try { $dsn = new DSN($connection); $device = $dsn->getScheme(); - } catch (\Exception $e) { - $device = STORAGE_DEVICE_LOCAL; + } catch (\Exception $e) {; } $buildId = $deployment->getAttribute('buildId', ''); diff --git a/composer.json b/composer.json index 2e6098c4c..680098038 100644 --- a/composer.json +++ b/composer.json @@ -62,7 +62,7 @@ "utopia-php/pools": "0.4.*", "utopia-php/preloader": "0.2.*", "utopia-php/registry": "0.5.*", - "utopia-php/storage": "0.11.*", + "utopia-php/storage": "0.13.*", "utopia-php/swoole": "0.5.*", "utopia-php/websocket": "0.1.0", "resque/php-resque": "1.3.6", diff --git a/composer.lock b/composer.lock index 54caaa313..309541d0c 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "9c2365876e32662c9c62e3a4c738fafe", + "content-hash": "2b8002030d115fd67f57b89a9972f104", "packages": [ { "name": "adhocore/jwt", @@ -2519,16 +2519,16 @@ }, { "name": "utopia-php/storage", - "version": "0.11.0", + "version": "0.13.0", "source": { "type": "git", "url": "https://github.com/utopia-php/storage.git", - "reference": "59802cf281d1976560cf6e353f250a9b870efddc" + "reference": "f34c010e4f8394a6b4aff70b6de55041d9a145d3" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/storage/zipball/59802cf281d1976560cf6e353f250a9b870efddc", - "reference": "59802cf281d1976560cf6e353f250a9b870efddc", + "url": "https://api.github.com/repos/utopia-php/storage/zipball/f34c010e4f8394a6b4aff70b6de55041d9a145d3", + "reference": "f34c010e4f8394a6b4aff70b6de55041d9a145d3", "shasum": "" }, "require": { @@ -2568,9 +2568,9 @@ ], "support": { "issues": "https://github.com/utopia-php/storage/issues", - "source": "https://github.com/utopia-php/storage/tree/0.11.0" + "source": "https://github.com/utopia-php/storage/tree/0.13.0" }, - "time": "2022-08-31T09:17:31+00:00" + "time": "2022-11-17T15:10:18+00:00" }, { "name": "utopia-php/swoole", diff --git a/src/Appwrite/Resque/Worker.php b/src/Appwrite/Resque/Worker.php index 5bf3e254e..f8e61c45d 100644 --- a/src/Appwrite/Resque/Worker.php +++ b/src/Appwrite/Resque/Worker.php @@ -19,6 +19,7 @@ use Utopia\Storage\Device\S3; use Utopia\Database\Document; use Utopia\Database\Validator\Authorization; use Utopia\DSN\DSN; +use Utopia\Storage\Storage; abstract class Worker { @@ -280,7 +281,7 @@ abstract class Worker $connection = App::getEnv('_APP_CONNECTIONS_STORAGE', ''); $acl = 'private'; - $device = STORAGE_DEVICE_LOCAL; + $device = Storage::DEVICE_LOCAL; $accessKey = ''; $accessSecret = ''; $bucket = ''; @@ -295,21 +296,21 @@ abstract class Worker $region = $dsn->getParam('region'); } catch (\Exception $e) { Console::error($e->getMessage() . 'Invalid DSN. Defaulting to Local storage.'); - $device = STORAGE_DEVICE_LOCAL; + $device = Storage::DEVICE_LOCAL; } switch ($device) { - case STORAGE_DEVICE_S3: + case Storage::DEVICE_S3: return new S3($root, $accessKey, $accessSecret, $bucket, $region, $acl); - case STORAGE_DEVICE_DO_SPACES: + case STORAGE::DEVICE_DO_SPACES: return new DOSpaces($root, $accessKey, $accessSecret, $bucket, $region, $acl); - case STORAGE_DEVICE_BACKBLAZE: + case Storage::DEVICE_BACKBLAZE: return new Backblaze($root, $accessKey, $accessSecret, $bucket, $region, $acl); - case STORAGE_DEVICE_LINODE: + case Storage::DEVICE_LINODE: return new Linode($root, $accessKey, $accessSecret, $bucket, $region, $acl); - case STORAGE_DEVICE_WASABI: + case Storage::DEVICE_WASABI: return new Wasabi($root, $accessKey, $accessSecret, $bucket, $region, $acl); - case STORAGE_DEVICE_LOCAL: + case Storage::DEVICE_LOCAL: default: return new Local($root); } From 6c43b44441b392bd4073d21a2c9da1d251e8aea5 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Thu, 17 Nov 2022 23:16:49 +0530 Subject: [PATCH 3/3] feat: use new storage library --- .env | 2 +- app/config/variables.php | 2 +- app/init.php | 3 +-- app/workers/builds.php | 3 ++- src/Appwrite/Resque/Worker.php | 3 +-- 5 files changed, 6 insertions(+), 7 deletions(-) diff --git a/.env b/.env index 58906ccd3..d2b567f68 100644 --- a/.env +++ b/.env @@ -29,7 +29,7 @@ _APP_CONNECTIONS_DB_CONSOLE=db_fra1_01=mariadb://user:password@mariadb:3306/appw _APP_CONNECTIONS_CACHE=redis_fra1_01=redis://redis:6379 _APP_CONNECTIONS_QUEUE=redis_fra1_01=redis://redis:6379 _APP_CONNECTIONS_PUBSUB=redis_fra1_01=redis://redis:6379 -_APP_CONNECTIONS_STORAGE=file://localhost +_APP_CONNECTIONS_STORAGE=local://localhost _APP_STORAGE_ANTIVIRUS=disabled _APP_STORAGE_ANTIVIRUS_HOST=clamav _APP_STORAGE_ANTIVIRUS_PORT=3310 diff --git a/app/config/variables.php b/app/config/variables.php index 9d5a0dd2e..e9f2e080d 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -509,7 +509,7 @@ return [ ], [ 'name' => '_APP_CONNECTIONS_STORAGE', - 'description' => 'A DSN representing the storage device to connect to. The DSN takes the following format ://:@:/?region=. For example, for S3: \'s3://access_key:access_secret@host:port/bucket?region=us-east-1\'. To use the local filesystem, you can leave this variable empty. Available devices are file, s3, dospaces, linode, backblaze and wasabi.', + 'description' => 'A DSN representing the storage device to connect to. The DSN takes the following format ://:@:/?region=. For example, for S3: \'s3://access_key:access_secret@host:port/bucket?region=us-east-1\'. To use the local filesystem, you can leave this variable empty. Available devices are local, s3, dospaces, linode, backblaze and wasabi.', 'introduction' => '1.1.0', 'default' => '', 'required' => false, diff --git a/app/init.php b/app/init.php index 323d931ad..9cfc579ba 100644 --- a/app/init.php +++ b/app/init.php @@ -1118,8 +1118,7 @@ function getDevice($root): Device $bucket = $dsn->getPath(); $region = $dsn->getParam('region'); } catch (\Exception $e) { - Console::error($e->getMessage() . 'Invalid DSN. Defaulting to Local storage.'); - $device = Storage::DEVICE_LOCAL; + Console::error($e->getMessage() . 'Invalid DSN. Defaulting to Local device.'); } switch ($device) { diff --git a/app/workers/builds.php b/app/workers/builds.php index 864987de7..12a4670b0 100644 --- a/app/workers/builds.php +++ b/app/workers/builds.php @@ -85,7 +85,8 @@ class BuildsV1 extends Worker try { $dsn = new DSN($connection); $device = $dsn->getScheme(); - } catch (\Exception $e) {; + } catch (\Exception $e) { + Console::error($e->getMessage() . 'Invalid DSN. Defaulting to Local device.'); } $buildId = $deployment->getAttribute('buildId', ''); diff --git a/src/Appwrite/Resque/Worker.php b/src/Appwrite/Resque/Worker.php index f8e61c45d..479855504 100644 --- a/src/Appwrite/Resque/Worker.php +++ b/src/Appwrite/Resque/Worker.php @@ -295,8 +295,7 @@ abstract class Worker $bucket = $dsn->getPath(); $region = $dsn->getParam('region'); } catch (\Exception $e) { - Console::error($e->getMessage() . 'Invalid DSN. Defaulting to Local storage.'); - $device = Storage::DEVICE_LOCAL; + Console::error($e->getMessage() . 'Invalid DSN. Defaulting to Local device.'); } switch ($device) {