From 9165da14b77a08b6bb92d699b099ee40450f1a19 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Wed, 26 Oct 2022 12:16:22 +0530 Subject: [PATCH 01/12] feat: add support for the new DSN class --- .env | 22 +----- Dockerfile | 22 +----- app/config/variables.php | 12 ++- app/controllers/api/storage.php | 2 +- app/executor.php | 53 ++++++------- app/init.php | 55 ++++++------- app/workers/builds.php | 2 +- composer.json | 5 ++ composer.lock | 65 +++++++++++++-- docker-compose.yml | 66 +--------------- src/Appwrite/DSN/DSN.php | 135 -------------------------------- src/Appwrite/Resque/Worker.php | 63 +++++++-------- tests/unit/DSN/DSNTest.php | 81 ------------------- 13 files changed, 151 insertions(+), 432 deletions(-) delete mode 100644 src/Appwrite/DSN/DSN.php delete mode 100644 tests/unit/DSN/DSNTest.php diff --git a/.env b/.env index 65fb54cb04..0ccd1d02f2 100644 --- a/.env +++ b/.env @@ -23,27 +23,7 @@ _APP_DB_SCHEMA=appwrite _APP_DB_USER=user _APP_DB_PASS=password _APP_DB_ROOT_PASS=rootsecretpassword -_APP_STORAGE_DEVICE=Local -_APP_STORAGE_S3_ACCESS_KEY= -_APP_STORAGE_S3_SECRET= -_APP_STORAGE_S3_REGION=us-east-1 -_APP_STORAGE_S3_BUCKET= -_APP_STORAGE_DO_SPACES_ACCESS_KEY= -_APP_STORAGE_DO_SPACES_SECRET= -_APP_STORAGE_DO_SPACES_REGION=us-east-1 -_APP_STORAGE_DO_SPACES_BUCKET= -_APP_STORAGE_BACKBLAZE_ACCESS_KEY= -_APP_STORAGE_BACKBLAZE_SECRET= -_APP_STORAGE_BACKBLAZE_REGION=us-west-004 -_APP_STORAGE_BACKBLAZE_BUCKET= -_APP_STORAGE_LINODE_ACCESS_KEY= -_APP_STORAGE_LINODE_SECRET= -_APP_STORAGE_LINODE_REGION=eu-central-1 -_APP_STORAGE_LINODE_BUCKET= -_APP_STORAGE_WASABI_ACCESS_KEY= -_APP_STORAGE_WASABI_SECRET= -_APP_STORAGE_WASABI_REGION=eu-central-1 -_APP_STORAGE_WASABI_BUCKET= +_APP_STORAGE_CONNECTION= _APP_STORAGE_ANTIVIRUS=disabled _APP_STORAGE_ANTIVIRUS_HOST=clamav _APP_STORAGE_ANTIVIRUS_PORT=3310 diff --git a/Dockerfile b/Dockerfile index a7cae38502..66a39584b2 100755 --- a/Dockerfile +++ b/Dockerfile @@ -195,27 +195,7 @@ ENV _APP_SERVER=swoole \ _APP_STORAGE_ANTIVIRUS=enabled \ _APP_STORAGE_ANTIVIRUS_HOST=clamav \ _APP_STORAGE_ANTIVIRUS_PORT=3310 \ - _APP_STORAGE_DEVICE=Local \ - _APP_STORAGE_S3_ACCESS_KEY= \ - _APP_STORAGE_S3_SECRET= \ - _APP_STORAGE_S3_REGION= \ - _APP_STORAGE_S3_BUCKET= \ - _APP_STORAGE_DO_SPACES_ACCESS_KEY= \ - _APP_STORAGE_DO_SPACES_SECRET= \ - _APP_STORAGE_DO_SPACES_REGION= \ - _APP_STORAGE_DO_SPACES_BUCKET= \ - _APP_STORAGE_BACKBLAZE_ACCESS_KEY= \ - _APP_STORAGE_BACKBLAZE_SECRET= \ - _APP_STORAGE_BACKBLAZE_REGION= \ - _APP_STORAGE_BACKBLAZE_BUCKET= \ - _APP_STORAGE_LINODE_ACCESS_KEY= \ - _APP_STORAGE_LINODE_SECRET= \ - _APP_STORAGE_LINODE_REGION= \ - _APP_STORAGE_LINODE_BUCKET= \ - _APP_STORAGE_WASABI_ACCESS_KEY= \ - _APP_STORAGE_WASABI_SECRET= \ - _APP_STORAGE_WASABI_REGION= \ - _APP_STORAGE_WASABI_BUCKET= \ + _APP_STORAGE_CONNECTION= \ _APP_REDIS_HOST=redis \ _APP_REDIS_PORT=6379 \ _APP_DB_HOST=mariadb \ diff --git a/app/config/variables.php b/app/config/variables.php index 9f3bc018e8..942de3bd13 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -481,16 +481,24 @@ return [ 'filter' => '' ], [ - 'name' => '_APP_STORAGE_DEVICE', + 'name' => '_APP_STORAGE_DEVICE', // TODO : Should we keep it or deprecate it considering we can get the device from the scheme of the DSN 'description' => 'Select default storage device. The default value is \'Local\'. List of supported adapters are \'Local\', \'S3\', \'DOSpaces\', \'Backblaze\', \'Linode\' and \'Wasabi\'.', 'introduction' => '0.13.0', 'default' => 'Local', 'required' => false, 'question' => '', ], + [ + 'name' => '_APP_STORAGE_CONNECTION', + 'description' => 'A DSN representing the storage device to connect to. The DSN takes the following format ://:@:/?region=. For example, for S3: \'s3://access_key:access_secret@host:port/bucket?region=us-east-1\'. To use the local filesystem, you can leave this variable empty.', + 'introduction' => '1.1.0', + 'default' => '', + 'required' => false, + 'question' => '', + ], [ 'name' => '_APP_STORAGE_S3_ACCESS_KEY', - 'description' => 'AWS S3 storage access key. Required when the storage adapter is set to S3. You can get your access key from your AWS console', + 'description' => 'Deprecated since 1.1.0. AWS S3 storage access key. Required when the storage adapter is set to S3. You can get your access key from your AWS console', 'introduction' => '0.13.0', 'default' => '', 'required' => false, diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index f236285749..0436a5f5be 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -494,7 +494,7 @@ App::post('/v1/storage/buckets/:bucketId/files') } if ($chunksUploaded === $chunks) { - if (App::getEnv('_APP_STORAGE_ANTIVIRUS') === 'enabled' && $bucket->getAttribute('antivirus', true) && $fileSize <= APP_LIMIT_ANTIVIRUS && App::getEnv('_APP_STORAGE_DEVICE', Storage::DEVICE_LOCAL) === Storage::DEVICE_LOCAL) { + if (App::getEnv('_APP_STORAGE_ANTIVIRUS') === 'enabled' && $bucket->getAttribute('antivirus', true) && $fileSize <= APP_LIMIT_ANTIVIRUS && App::getEnv('_APP_STORAGE_DEVICE', Storage::DEVICE_LOCAL) === Storage::DEVICE_LOCAL) { // TODO : fetch from DSN $antivirus = new Network( App::getEnv('_APP_STORAGE_ANTIVIRUS_HOST', 'clamav'), (int) App::getEnv('_APP_STORAGE_ANTIVIRUS_PORT', 3310) diff --git a/app/executor.php b/app/executor.php index fba8c4c416..a09de2632b 100644 --- a/app/executor.php +++ b/app/executor.php @@ -13,6 +13,7 @@ use Swoole\Timer; use Utopia\App; use Utopia\CLI\Console; use Utopia\Database\DateTime; +use Utopia\DSN\DSN; use Utopia\Logger\Log; use Utopia\Logger\Logger; use Utopia\Orchestration\Adapter\DockerCLI; @@ -119,45 +120,35 @@ function logError(Throwable $error, string $action, Utopia\Route $route = null) function getStorageDevice($root): Device { - switch (App::getEnv('_APP_STORAGE_DEVICE', Storage::DEVICE_LOCAL)) { + $connection = App::getEnv('_APP_STORAGE_CONNECTION', ''); + + try { + $dsn = new DSN($connection); + $device = $dsn->getScheme(); + $accessKey = $dsn->getUser(); + $accessSecret = $dsn->getPassword(); + $bucket = $dsn->getPath(); + $region = $dsn->getParam('region'); + $acl = 'private'; + } catch (\Exception $e) { + Console::error($e->getMessage() . 'Defaulting to Local storage.'); + $device = 'Local'; + } + + switch ($device) { case Storage::DEVICE_LOCAL: default: return new Local($root); case Storage::DEVICE_S3: - $s3AccessKey = App::getEnv('_APP_STORAGE_S3_ACCESS_KEY', ''); - $s3SecretKey = App::getEnv('_APP_STORAGE_S3_SECRET', ''); - $s3Region = App::getEnv('_APP_STORAGE_S3_REGION', ''); - $s3Bucket = App::getEnv('_APP_STORAGE_S3_BUCKET', ''); - $s3Acl = 'private'; - return new S3($root, $s3AccessKey, $s3SecretKey, $s3Bucket, $s3Region, $s3Acl); + return new S3($root, $accessKey, $accessSecret, $bucket, $region, $acl); case Storage::DEVICE_DO_SPACES: - $doSpacesAccessKey = App::getEnv('_APP_STORAGE_DO_SPACES_ACCESS_KEY', ''); - $doSpacesSecretKey = App::getEnv('_APP_STORAGE_DO_SPACES_SECRET', ''); - $doSpacesRegion = App::getEnv('_APP_STORAGE_DO_SPACES_REGION', ''); - $doSpacesBucket = App::getEnv('_APP_STORAGE_DO_SPACES_BUCKET', ''); - $doSpacesAcl = 'private'; - return new DOSpaces($root, $doSpacesAccessKey, $doSpacesSecretKey, $doSpacesBucket, $doSpacesRegion, $doSpacesAcl); + return new DOSpaces($root, $accessKey, $accessSecret, $bucket, $region, $acl); case Storage::DEVICE_BACKBLAZE: - $backblazeAccessKey = App::getEnv('_APP_STORAGE_BACKBLAZE_ACCESS_KEY', ''); - $backblazeSecretKey = App::getEnv('_APP_STORAGE_BACKBLAZE_SECRET', ''); - $backblazeRegion = App::getEnv('_APP_STORAGE_BACKBLAZE_REGION', ''); - $backblazeBucket = App::getEnv('_APP_STORAGE_BACKBLAZE_BUCKET', ''); - $backblazeAcl = 'private'; - return new Backblaze($root, $backblazeAccessKey, $backblazeSecretKey, $backblazeBucket, $backblazeRegion, $backblazeAcl); + return new Backblaze($root, $accessKey, $accessSecret, $bucket, $region, $acl); case Storage::DEVICE_LINODE: - $linodeAccessKey = App::getEnv('_APP_STORAGE_LINODE_ACCESS_KEY', ''); - $linodeSecretKey = App::getEnv('_APP_STORAGE_LINODE_SECRET', ''); - $linodeRegion = App::getEnv('_APP_STORAGE_LINODE_REGION', ''); - $linodeBucket = App::getEnv('_APP_STORAGE_LINODE_BUCKET', ''); - $linodeAcl = 'private'; - return new Linode($root, $linodeAccessKey, $linodeSecretKey, $linodeBucket, $linodeRegion, $linodeAcl); + return new Linode($root, $accessKey, $accessSecret, $bucket, $region, $acl); case Storage::DEVICE_WASABI: - $wasabiAccessKey = App::getEnv('_APP_STORAGE_WASABI_ACCESS_KEY', ''); - $wasabiSecretKey = App::getEnv('_APP_STORAGE_WASABI_SECRET', ''); - $wasabiRegion = App::getEnv('_APP_STORAGE_WASABI_REGION', ''); - $wasabiBucket = App::getEnv('_APP_STORAGE_WASABI_BUCKET', ''); - $wasabiAcl = 'private'; - return new Wasabi($root, $wasabiAccessKey, $wasabiSecretKey, $wasabiBucket, $wasabiRegion, $wasabiAcl); + return new Wasabi($root, $accessKey, $accessSecret, $bucket, $region, $acl); } } diff --git a/app/init.php b/app/init.php index 054721d439..cde78c4ce9 100644 --- a/app/init.php +++ b/app/init.php @@ -29,7 +29,6 @@ use Appwrite\SMS\Adapter\TextMagic; use Appwrite\SMS\Adapter\Twilio; use Appwrite\SMS\Adapter\Msg91; use Appwrite\SMS\Adapter\Vonage; -use Appwrite\DSN\DSN; use Appwrite\Event\Audit; use Appwrite\Event\Database as EventDatabase; use Appwrite\Event\Delete; @@ -63,8 +62,10 @@ use Swoole\Database\PDOConfig; use Swoole\Database\PDOPool; use Swoole\Database\RedisConfig; use Swoole\Database\RedisPool; +use Utopia\CLI\Console; use Utopia\Database\Query; use Utopia\Database\Validator\DatetimeValidator; +use Utopia\DSN\DSN; use Utopia\Storage\Device; use Utopia\Storage\Storage; use Utopia\Storage\Device\Backblaze; @@ -964,45 +965,35 @@ App::setResource('deviceBuilds', function ($project) { function getDevice($root): Device { - switch (App::getEnv('_APP_STORAGE_DEVICE', Storage::DEVICE_LOCAL)) { + $connection = App::getEnv('_APP_STORAGE_CONNECTION', ''); + + try { + $dsn = new DSN($connection); + $device = $dsn->getScheme(); + $accessKey = $dsn->getUser(); + $accessSecret = $dsn->getPassword(); + $bucket = $dsn->getPath(); + $region = $dsn->getParam('region'); + $acl = 'private'; + } catch (\Exception $e) { + Console::error($e->getMessage() . 'Defaulting to Local storage.'); + $device = 'Local'; + } + + switch ($device) { case Storage::DEVICE_LOCAL: default: return new Local($root); case Storage::DEVICE_S3: - $s3AccessKey = App::getEnv('_APP_STORAGE_S3_ACCESS_KEY', ''); - $s3SecretKey = App::getEnv('_APP_STORAGE_S3_SECRET', ''); - $s3Region = App::getEnv('_APP_STORAGE_S3_REGION', ''); - $s3Bucket = App::getEnv('_APP_STORAGE_S3_BUCKET', ''); - $s3Acl = 'private'; - return new S3($root, $s3AccessKey, $s3SecretKey, $s3Bucket, $s3Region, $s3Acl); + return new S3($root, $accessKey, $accessSecret, $bucket, $region, $acl); case Storage::DEVICE_DO_SPACES: - $doSpacesAccessKey = App::getEnv('_APP_STORAGE_DO_SPACES_ACCESS_KEY', ''); - $doSpacesSecretKey = App::getEnv('_APP_STORAGE_DO_SPACES_SECRET', ''); - $doSpacesRegion = App::getEnv('_APP_STORAGE_DO_SPACES_REGION', ''); - $doSpacesBucket = App::getEnv('_APP_STORAGE_DO_SPACES_BUCKET', ''); - $doSpacesAcl = 'private'; - return new DOSpaces($root, $doSpacesAccessKey, $doSpacesSecretKey, $doSpacesBucket, $doSpacesRegion, $doSpacesAcl); + return new DOSpaces($root, $accessKey, $accessSecret, $bucket, $region, $acl); case Storage::DEVICE_BACKBLAZE: - $backblazeAccessKey = App::getEnv('_APP_STORAGE_BACKBLAZE_ACCESS_KEY', ''); - $backblazeSecretKey = App::getEnv('_APP_STORAGE_BACKBLAZE_SECRET', ''); - $backblazeRegion = App::getEnv('_APP_STORAGE_BACKBLAZE_REGION', ''); - $backblazeBucket = App::getEnv('_APP_STORAGE_BACKBLAZE_BUCKET', ''); - $backblazeAcl = 'private'; - return new Backblaze($root, $backblazeAccessKey, $backblazeSecretKey, $backblazeBucket, $backblazeRegion, $backblazeAcl); + return new Backblaze($root, $accessKey, $accessSecret, $bucket, $region, $acl); case Storage::DEVICE_LINODE: - $linodeAccessKey = App::getEnv('_APP_STORAGE_LINODE_ACCESS_KEY', ''); - $linodeSecretKey = App::getEnv('_APP_STORAGE_LINODE_SECRET', ''); - $linodeRegion = App::getEnv('_APP_STORAGE_LINODE_REGION', ''); - $linodeBucket = App::getEnv('_APP_STORAGE_LINODE_BUCKET', ''); - $linodeAcl = 'private'; - return new Linode($root, $linodeAccessKey, $linodeSecretKey, $linodeBucket, $linodeRegion, $linodeAcl); + return new Linode($root, $accessKey, $accessSecret, $bucket, $region, $acl); case Storage::DEVICE_WASABI: - $wasabiAccessKey = App::getEnv('_APP_STORAGE_WASABI_ACCESS_KEY', ''); - $wasabiSecretKey = App::getEnv('_APP_STORAGE_WASABI_SECRET', ''); - $wasabiRegion = App::getEnv('_APP_STORAGE_WASABI_REGION', ''); - $wasabiBucket = App::getEnv('_APP_STORAGE_WASABI_BUCKET', ''); - $wasabiAcl = 'private'; - return new Wasabi($root, $wasabiAccessKey, $wasabiSecretKey, $wasabiBucket, $wasabiRegion, $wasabiAcl); + return new Wasabi($root, $accessKey, $accessSecret, $bucket, $region, $acl); } } diff --git a/app/workers/builds.php b/app/workers/builds.php index bf780c6464..f8ffb8a8f0 100644 --- a/app/workers/builds.php +++ b/app/workers/builds.php @@ -91,7 +91,7 @@ class BuildsV1 extends Worker 'outputPath' => '', 'runtime' => $function->getAttribute('runtime'), 'source' => $deployment->getAttribute('path'), - 'sourceType' => App::getEnv('_APP_STORAGE_DEVICE', Storage::DEVICE_LOCAL), + 'sourceType' => App::getEnv('_APP_STORAGE_DEVICE', Storage::DEVICE_LOCAL), // TODO : fetch from DSN 'stdout' => '', 'stderr' => '', 'endTime' => null, diff --git a/composer.json b/composer.json index 47317ed5e2..82149d15c3 100644 --- a/composer.json +++ b/composer.json @@ -57,6 +57,7 @@ "utopia-php/preloader": "0.2.*", "utopia-php/domains": "1.1.*", "utopia-php/swoole": "0.3.*", + "utopia-php/dsn": "dev-dev", "utopia-php/storage": "0.11.*", "utopia-php/websocket": "0.1.0", "utopia-php/image": "0.5.*", @@ -74,6 +75,10 @@ { "url": "https://github.com/appwrite/runtimes.git", "type": "git" + }, + { + "url": "https://github.com/utopia-php/dsn.git", + "type": "git" } ], "require-dev": { diff --git a/composer.lock b/composer.lock index f6f29452ad..3f3f4833b6 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "08fdd139ad1285b02c4b4e555679e7de", + "content-hash": "b9786b80c04af9966f4d36c085dd4090", "packages": [ { "name": "adhocore/jwt", @@ -693,16 +693,16 @@ }, { "name": "guzzlehttp/psr7", - "version": "2.4.1", + "version": "2.4.2", "source": { "type": "git", "url": "https://github.com/guzzle/psr7.git", - "reference": "69568e4293f4fa993f3b0e51c9723e1e17c41379" + "reference": "3148458748274be1546f8f2809a6c09fe66f44aa" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/guzzle/psr7/zipball/69568e4293f4fa993f3b0e51c9723e1e17c41379", - "reference": "69568e4293f4fa993f3b0e51c9723e1e17c41379", + "url": "https://api.github.com/repos/guzzle/psr7/zipball/3148458748274be1546f8f2809a6c09fe66f44aa", + "reference": "3148458748274be1546f8f2809a6c09fe66f44aa", "shasum": "" }, "require": { @@ -792,7 +792,7 @@ ], "support": { "issues": "https://github.com/guzzle/psr7/issues", - "source": "https://github.com/guzzle/psr7/tree/2.4.1" + "source": "https://github.com/guzzle/psr7/tree/2.4.2" }, "funding": [ { @@ -808,7 +808,7 @@ "type": "tidelift" } ], - "time": "2022-08-28T14:45:39+00:00" + "time": "2022-10-25T13:49:28+00:00" }, { "name": "influxdb/influxdb-php", @@ -2170,6 +2170,53 @@ }, "time": "2020-02-23T07:40:02+00:00" }, + { + "name": "utopia-php/dsn", + "version": "dev-dev", + "source": { + "type": "git", + "url": "https://github.com/utopia-php/dsn.git", + "reference": "9a524236c37474fcb121dc0d9eb13a0bcead9cf6" + }, + "require": { + "php": ">=8.0" + }, + "require-dev": { + "laravel/pint": "1.2.*", + "phpunit/phpunit": "^9.3", + "squizlabs/php_codesniffer": "^3.6", + "vimeo/psalm": "4.0.1" + }, + "type": "library", + "autoload": { + "psr-4": { + "Utopia\\DSN\\": "src/DSN" + } + }, + "scripts": { + "lint": [ + "./vendor/bin/pint --test" + ], + "format": [ + "./vendor/bin/pint" + ], + "test": [ + "./vendor/bin/phpunit --configuration phpunit.xml tests" + ] + }, + "license": [ + "MIT" + ], + "description": "A simple library for parsing and managing Data Source Names ( DSNs )", + "keywords": [ + "dsn", + "framework", + "php", + "upf", + "utopia" + ], + "time": "2022-10-26T03:49:15+00:00" + }, { "name": "utopia-php/framework", "version": "0.21.1", @@ -5359,7 +5406,9 @@ ], "aliases": [], "minimum-stability": "stable", - "stability-flags": [], + "stability-flags": { + "utopia-php/dsn": 20 + }, "prefer-stable": false, "prefer-lowest": false, "platform": { diff --git a/docker-compose.yml b/docker-compose.yml index ea9241a9d3..87bf5d5efb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -131,27 +131,7 @@ services: - _APP_STORAGE_ANTIVIRUS - _APP_STORAGE_ANTIVIRUS_HOST - _APP_STORAGE_ANTIVIRUS_PORT - - _APP_STORAGE_DEVICE - - _APP_STORAGE_S3_ACCESS_KEY - - _APP_STORAGE_S3_SECRET - - _APP_STORAGE_S3_REGION - - _APP_STORAGE_S3_BUCKET - - _APP_STORAGE_DO_SPACES_ACCESS_KEY - - _APP_STORAGE_DO_SPACES_SECRET - - _APP_STORAGE_DO_SPACES_REGION - - _APP_STORAGE_DO_SPACES_BUCKET - - _APP_STORAGE_BACKBLAZE_ACCESS_KEY - - _APP_STORAGE_BACKBLAZE_SECRET - - _APP_STORAGE_BACKBLAZE_REGION - - _APP_STORAGE_BACKBLAZE_BUCKET - - _APP_STORAGE_LINODE_ACCESS_KEY - - _APP_STORAGE_LINODE_SECRET - - _APP_STORAGE_LINODE_REGION - - _APP_STORAGE_LINODE_BUCKET - - _APP_STORAGE_WASABI_ACCESS_KEY - - _APP_STORAGE_WASABI_SECRET - - _APP_STORAGE_WASABI_REGION - - _APP_STORAGE_WASABI_BUCKET + - _APP_STORAGE_CONNECTION - _APP_FUNCTIONS_SIZE_LIMIT - _APP_FUNCTIONS_TIMEOUT - _APP_FUNCTIONS_BUILD_TIMEOUT @@ -304,27 +284,7 @@ services: - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS - - _APP_STORAGE_DEVICE - - _APP_STORAGE_S3_ACCESS_KEY - - _APP_STORAGE_S3_SECRET - - _APP_STORAGE_S3_REGION - - _APP_STORAGE_S3_BUCKET - - _APP_STORAGE_DO_SPACES_ACCESS_KEY - - _APP_STORAGE_DO_SPACES_SECRET - - _APP_STORAGE_DO_SPACES_REGION - - _APP_STORAGE_DO_SPACES_BUCKET - - _APP_STORAGE_BACKBLAZE_ACCESS_KEY - - _APP_STORAGE_BACKBLAZE_SECRET - - _APP_STORAGE_BACKBLAZE_REGION - - _APP_STORAGE_BACKBLAZE_BUCKET - - _APP_STORAGE_LINODE_ACCESS_KEY - - _APP_STORAGE_LINODE_SECRET - - _APP_STORAGE_LINODE_REGION - - _APP_STORAGE_LINODE_BUCKET - - _APP_STORAGE_WASABI_ACCESS_KEY - - _APP_STORAGE_WASABI_SECRET - - _APP_STORAGE_WASABI_REGION - - _APP_STORAGE_WASABI_BUCKET + - _APP_STORAGE_CONNECTION - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG - _APP_EXECUTOR_SECRET @@ -492,27 +452,7 @@ services: - OPEN_RUNTIMES_NETWORK - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG - - _APP_STORAGE_DEVICE - - _APP_STORAGE_S3_ACCESS_KEY - - _APP_STORAGE_S3_SECRET - - _APP_STORAGE_S3_REGION - - _APP_STORAGE_S3_BUCKET - - _APP_STORAGE_DO_SPACES_ACCESS_KEY - - _APP_STORAGE_DO_SPACES_SECRET - - _APP_STORAGE_DO_SPACES_REGION - - _APP_STORAGE_DO_SPACES_BUCKET - - _APP_STORAGE_BACKBLAZE_ACCESS_KEY - - _APP_STORAGE_BACKBLAZE_SECRET - - _APP_STORAGE_BACKBLAZE_REGION - - _APP_STORAGE_BACKBLAZE_BUCKET - - _APP_STORAGE_LINODE_ACCESS_KEY - - _APP_STORAGE_LINODE_SECRET - - _APP_STORAGE_LINODE_REGION - - _APP_STORAGE_LINODE_BUCKET - - _APP_STORAGE_WASABI_ACCESS_KEY - - _APP_STORAGE_WASABI_SECRET - - _APP_STORAGE_WASABI_REGION - - _APP_STORAGE_WASABI_BUCKET + - _APP_STORAGE_CONNECTION - DOCKERHUB_PULL_USERNAME - DOCKERHUB_PULL_PASSWORD diff --git a/src/Appwrite/DSN/DSN.php b/src/Appwrite/DSN/DSN.php deleted file mode 100644 index 5605640989..0000000000 --- a/src/Appwrite/DSN/DSN.php +++ /dev/null @@ -1,135 +0,0 @@ -scheme = $parts['scheme'] ?? null; - $this->user = $parts['user'] ?? null; - $this->password = $parts['pass'] ?? null; - $this->host = $parts['host'] ?? null; - $this->port = $parts['port'] ?? null; - $this->database = $parts['path'] ?? null; - $this->query = $parts['query'] ?? null; - } - - /** - * Return the scheme. - * - * @return string - */ - public function getScheme(): string - { - return $this->scheme; - } - - /** - * Return the user. - * - * @return ?string - */ - public function getUser(): ?string - { - return $this->user; - } - - /** - * Return the password. - * - * @return ?string - */ - public function getPassword(): ?string - { - return $this->password; - } - - /** - * Return the host - * - * @return string - */ - public function getHost(): string - { - return $this->host; - } - - /** - * Return the port - * - * @return ?string - */ - public function getPort(): ?string - { - return $this->port; - } - - /** - * Return the database - * - * @return ?string - */ - public function getDatabase(): ?string - { - return ltrim($this->database, '/'); - } - - /** - * Return the query string - * - * @return ?string - */ - public function getQuery(): ?string - { - return $this->query; - } -} diff --git a/src/Appwrite/Resque/Worker.php b/src/Appwrite/Resque/Worker.php index dd7cebd084..c6e5e90c5e 100644 --- a/src/Appwrite/Resque/Worker.php +++ b/src/Appwrite/Resque/Worker.php @@ -18,6 +18,7 @@ use Utopia\Storage\Device\Backblaze; use Utopia\Storage\Device\S3; use Exception; use Utopia\Database\Validator\Authorization; +use Utopia\DSN\DSN; abstract class Worker { @@ -284,45 +285,35 @@ abstract class Worker */ public function getDevice($root): Device { - switch (App::getEnv('_APP_STORAGE_DEVICE', Storage::DEVICE_LOCAL)) { + $connection = App::getEnv('_APP_STORAGE_CONNECTION', ''); + + try { + $dsn = new DSN($connection); + $device = $dsn->getScheme(); + $accessKey = $dsn->getUser(); + $accessSecret = $dsn->getPassword(); + $bucket = $dsn->getPath(); + $region = $dsn->getParam('region'); + $acl = 'private'; + } catch (\Exception $e) { + Console::error($e->getMessage() . 'Defaulting to Local storage.'); + $device = 'Local'; + } + + switch ($device) { + case Storage::DEVICE_S3: + return new S3($root, $accessKey, $accessSecret, $bucket, $region, $acl); + case Storage::DEVICE_DO_SPACES: + return new DOSpaces($root, $accessKey, $accessSecret, $bucket, $region, $acl); + case Storage::DEVICE_BACKBLAZE: + return new Backblaze($root, $accessKey, $accessSecret, $bucket, $region, $acl); + case Storage::DEVICE_LINODE: + return new Linode($root, $accessKey, $accessSecret, $bucket, $region, $acl); + case Storage::DEVICE_WASABI: + return new Wasabi($root, $accessKey, $accessSecret, $bucket, $region, $acl); case Storage::DEVICE_LOCAL: default: return new Local($root); - case Storage::DEVICE_S3: - $s3AccessKey = App::getEnv('_APP_STORAGE_S3_ACCESS_KEY', ''); - $s3SecretKey = App::getEnv('_APP_STORAGE_S3_SECRET', ''); - $s3Region = App::getEnv('_APP_STORAGE_S3_REGION', ''); - $s3Bucket = App::getEnv('_APP_STORAGE_S3_BUCKET', ''); - $s3Acl = 'private'; - return new S3($root, $s3AccessKey, $s3SecretKey, $s3Bucket, $s3Region, $s3Acl); - case Storage::DEVICE_DO_SPACES: - $doSpacesAccessKey = App::getEnv('_APP_STORAGE_DO_SPACES_ACCESS_KEY', ''); - $doSpacesSecretKey = App::getEnv('_APP_STORAGE_DO_SPACES_SECRET', ''); - $doSpacesRegion = App::getEnv('_APP_STORAGE_DO_SPACES_REGION', ''); - $doSpacesBucket = App::getEnv('_APP_STORAGE_DO_SPACES_BUCKET', ''); - $doSpacesAcl = 'private'; - return new DOSpaces($root, $doSpacesAccessKey, $doSpacesSecretKey, $doSpacesBucket, $doSpacesRegion, $doSpacesAcl); - case Storage::DEVICE_BACKBLAZE: - $backblazeAccessKey = App::getEnv('_APP_STORAGE_BACKBLAZE_ACCESS_KEY', ''); - $backblazeSecretKey = App::getEnv('_APP_STORAGE_BACKBLAZE_SECRET', ''); - $backblazeRegion = App::getEnv('_APP_STORAGE_BACKBLAZE_REGION', ''); - $backblazeBucket = App::getEnv('_APP_STORAGE_BACKBLAZE_BUCKET', ''); - $backblazeAcl = 'private'; - return new Backblaze($root, $backblazeAccessKey, $backblazeSecretKey, $backblazeBucket, $backblazeRegion, $backblazeAcl); - case Storage::DEVICE_LINODE: - $linodeAccessKey = App::getEnv('_APP_STORAGE_LINODE_ACCESS_KEY', ''); - $linodeSecretKey = App::getEnv('_APP_STORAGE_LINODE_SECRET', ''); - $linodeRegion = App::getEnv('_APP_STORAGE_LINODE_REGION', ''); - $linodeBucket = App::getEnv('_APP_STORAGE_LINODE_BUCKET', ''); - $linodeAcl = 'private'; - return new Linode($root, $linodeAccessKey, $linodeSecretKey, $linodeBucket, $linodeRegion, $linodeAcl); - case Storage::DEVICE_WASABI: - $wasabiAccessKey = App::getEnv('_APP_STORAGE_WASABI_ACCESS_KEY', ''); - $wasabiSecretKey = App::getEnv('_APP_STORAGE_WASABI_SECRET', ''); - $wasabiRegion = App::getEnv('_APP_STORAGE_WASABI_REGION', ''); - $wasabiBucket = App::getEnv('_APP_STORAGE_WASABI_BUCKET', ''); - $wasabiAcl = 'private'; - return new Wasabi($root, $wasabiAccessKey, $wasabiSecretKey, $wasabiBucket, $wasabiRegion, $wasabiAcl); } } } diff --git a/tests/unit/DSN/DSNTest.php b/tests/unit/DSN/DSNTest.php deleted file mode 100644 index d1f5ba1197..0000000000 --- a/tests/unit/DSN/DSNTest.php +++ /dev/null @@ -1,81 +0,0 @@ -assertEquals("mariadb", $dsn->getScheme()); - $this->assertEquals("user", $dsn->getUser()); - $this->assertEquals("password", $dsn->getPassword()); - $this->assertEquals("localhost", $dsn->getHost()); - $this->assertEquals("3306", $dsn->getPort()); - $this->assertEquals("database", $dsn->getDatabase()); - $this->assertEquals("charset=utf8&timezone=UTC", $dsn->getQuery()); - - $dsn = new DSN("mariadb://user@localhost:3306/database?charset=utf8&timezone=UTC"); - $this->assertEquals("mariadb", $dsn->getScheme()); - $this->assertEquals("user", $dsn->getUser()); - $this->assertNull($dsn->getPassword()); - $this->assertEquals("localhost", $dsn->getHost()); - $this->assertEquals("3306", $dsn->getPort()); - $this->assertEquals("database", $dsn->getDatabase()); - $this->assertEquals("charset=utf8&timezone=UTC", $dsn->getQuery()); - - $dsn = new DSN("mariadb://user@localhost/database?charset=utf8&timezone=UTC"); - $this->assertEquals("mariadb", $dsn->getScheme()); - $this->assertEquals("user", $dsn->getUser()); - $this->assertNull($dsn->getPassword()); - $this->assertEquals("localhost", $dsn->getHost()); - $this->assertNull($dsn->getPort()); - $this->assertEquals("database", $dsn->getDatabase()); - $this->assertEquals("charset=utf8&timezone=UTC", $dsn->getQuery()); - - $dsn = new DSN("mariadb://user@localhost?charset=utf8&timezone=UTC"); - $this->assertEquals("mariadb", $dsn->getScheme()); - $this->assertEquals("user", $dsn->getUser()); - $this->assertNull($dsn->getPassword()); - $this->assertEquals("localhost", $dsn->getHost()); - $this->assertNull($dsn->getPort()); - $this->assertEmpty($dsn->getDatabase()); - $this->assertEquals("charset=utf8&timezone=UTC", $dsn->getQuery()); - - $dsn = new DSN("mariadb://user@localhost"); - $this->assertEquals("mariadb", $dsn->getScheme()); - $this->assertEquals("user", $dsn->getUser()); - $this->assertNull($dsn->getPassword()); - $this->assertEquals("localhost", $dsn->getHost()); - $this->assertNull($dsn->getPort()); - $this->assertEmpty($dsn->getDatabase()); - $this->assertNull($dsn->getQuery()); - - $dsn = new DSN("mariadb://user:@localhost"); - $this->assertEquals("mariadb", $dsn->getScheme()); - $this->assertEquals("user", $dsn->getUser()); - $this->assertEmpty($dsn->getPassword()); - $this->assertEquals("localhost", $dsn->getHost()); - $this->assertNull($dsn->getPort()); - $this->assertEmpty($dsn->getDatabase()); - $this->assertNull($dsn->getQuery()); - - $dsn = new DSN("mariadb://localhost"); - $this->assertEquals("mariadb", $dsn->getScheme()); - $this->assertNull($dsn->getUser()); - $this->assertNull($dsn->getPassword()); - $this->assertEquals("localhost", $dsn->getHost()); - $this->assertNull($dsn->getPort()); - $this->assertEmpty($dsn->getDatabase()); - $this->assertNull($dsn->getQuery()); - } - - public function testFail(): void - { - $this->expectException(\InvalidArgumentException::class); - new DSN("mariadb://"); - } -} From d75831380ba125c0d5278815b654e791295debaf Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Thu, 17 Nov 2022 06:36:41 +0000 Subject: [PATCH 02/12] feat: review comments --- .env | 2 +- Dockerfile | 2 +- app/config/variables.php | 2 +- app/executor.php | 2 +- app/init.php | 2 +- composer.lock | 182 ++++++++++++++++++++++----------- docker-compose.yml | 6 +- src/Appwrite/Resque/Worker.php | 2 +- 8 files changed, 134 insertions(+), 66 deletions(-) diff --git a/.env b/.env index 0ccd1d02f2..3e9c4f2660 100644 --- a/.env +++ b/.env @@ -23,7 +23,7 @@ _APP_DB_SCHEMA=appwrite _APP_DB_USER=user _APP_DB_PASS=password _APP_DB_ROOT_PASS=rootsecretpassword -_APP_STORAGE_CONNECTION= +_APP_CONNECTIONS_STORAGE= _APP_STORAGE_ANTIVIRUS=disabled _APP_STORAGE_ANTIVIRUS_HOST=clamav _APP_STORAGE_ANTIVIRUS_PORT=3310 diff --git a/Dockerfile b/Dockerfile index 66a39584b2..813c1b16ab 100755 --- a/Dockerfile +++ b/Dockerfile @@ -195,7 +195,7 @@ ENV _APP_SERVER=swoole \ _APP_STORAGE_ANTIVIRUS=enabled \ _APP_STORAGE_ANTIVIRUS_HOST=clamav \ _APP_STORAGE_ANTIVIRUS_PORT=3310 \ - _APP_STORAGE_CONNECTION= \ + _APP_CONNECTIONS_STORAGE= \ _APP_REDIS_HOST=redis \ _APP_REDIS_PORT=6379 \ _APP_DB_HOST=mariadb \ diff --git a/app/config/variables.php b/app/config/variables.php index 942de3bd13..ecbe5115ce 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -489,7 +489,7 @@ return [ 'question' => '', ], [ - 'name' => '_APP_STORAGE_CONNECTION', + 'name' => '_APP_CONNECTIONS_STORAGE', 'description' => 'A DSN representing the storage device to connect to. The DSN takes the following format ://:@:/?region=. For example, for S3: \'s3://access_key:access_secret@host:port/bucket?region=us-east-1\'. To use the local filesystem, you can leave this variable empty.', 'introduction' => '1.1.0', 'default' => '', diff --git a/app/executor.php b/app/executor.php index a09de2632b..68dc207da1 100644 --- a/app/executor.php +++ b/app/executor.php @@ -120,7 +120,7 @@ function logError(Throwable $error, string $action, Utopia\Route $route = null) function getStorageDevice($root): Device { - $connection = App::getEnv('_APP_STORAGE_CONNECTION', ''); + $connection = App::getEnv('_APP_CONNECTIONS_STORAGE', ''); try { $dsn = new DSN($connection); diff --git a/app/init.php b/app/init.php index cde78c4ce9..4645b8720f 100644 --- a/app/init.php +++ b/app/init.php @@ -965,7 +965,7 @@ App::setResource('deviceBuilds', function ($project) { function getDevice($root): Device { - $connection = App::getEnv('_APP_STORAGE_CONNECTION', ''); + $connection = App::getEnv('_APP_CONNECTIONS_STORAGE', ''); try { $dsn = new DSN($connection); diff --git a/composer.lock b/composer.lock index 3f3f4833b6..dd5721f219 100644 --- a/composer.lock +++ b/composer.lock @@ -115,15 +115,15 @@ }, { "name": "appwrite/php-runtimes", - "version": "0.11.0", + "version": "0.11.1", "source": { "type": "git", "url": "https://github.com/appwrite/runtimes.git", - "reference": "547fc026e11c0946846a8ac690898f5bf53be101" + "reference": "9d74a477ba3333cbcfac565c46fcf19606b7b603" }, "require": { "php": ">=8.0", - "utopia-php/system": "0.4.*" + "utopia-php/system": "0.6.*" }, "require-dev": { "phpunit/phpunit": "^9.3", @@ -154,7 +154,7 @@ "php", "runtimes" ], - "time": "2022-08-15T14:03:36+00:00" + "time": "2022-11-07T16:45:52+00:00" }, { "name": "chillerlan/php-qrcode", @@ -300,16 +300,16 @@ }, { "name": "colinmollenhour/credis", - "version": "v1.13.1", + "version": "v1.14.0", "source": { "type": "git", "url": "https://github.com/colinmollenhour/credis.git", - "reference": "85df015088e00daf8ce395189de22c8eb45c8d49" + "reference": "dccc8a46586475075fbb012d8bd523b8a938c2dc" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/colinmollenhour/credis/zipball/85df015088e00daf8ce395189de22c8eb45c8d49", - "reference": "85df015088e00daf8ce395189de22c8eb45c8d49", + "url": "https://api.github.com/repos/colinmollenhour/credis/zipball/dccc8a46586475075fbb012d8bd523b8a938c2dc", + "reference": "dccc8a46586475075fbb012d8bd523b8a938c2dc", "shasum": "" }, "require": { @@ -341,9 +341,9 @@ "homepage": "https://github.com/colinmollenhour/credis", "support": { "issues": "https://github.com/colinmollenhour/credis/issues", - "source": "https://github.com/colinmollenhour/credis/tree/v1.13.1" + "source": "https://github.com/colinmollenhour/credis/tree/v1.14.0" }, - "time": "2022-06-20T22:56:59+00:00" + "time": "2022-11-09T01:18:39+00:00" }, { "name": "composer/package-versions-deprecated", @@ -693,16 +693,16 @@ }, { "name": "guzzlehttp/psr7", - "version": "2.4.2", + "version": "2.4.3", "source": { "type": "git", "url": "https://github.com/guzzle/psr7.git", - "reference": "3148458748274be1546f8f2809a6c09fe66f44aa" + "reference": "67c26b443f348a51926030c83481b85718457d3d" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/guzzle/psr7/zipball/3148458748274be1546f8f2809a6c09fe66f44aa", - "reference": "3148458748274be1546f8f2809a6c09fe66f44aa", + "url": "https://api.github.com/repos/guzzle/psr7/zipball/67c26b443f348a51926030c83481b85718457d3d", + "reference": "67c26b443f348a51926030c83481b85718457d3d", "shasum": "" }, "require": { @@ -792,7 +792,7 @@ ], "support": { "issues": "https://github.com/guzzle/psr7/issues", - "source": "https://github.com/guzzle/psr7/tree/2.4.2" + "source": "https://github.com/guzzle/psr7/tree/2.4.3" }, "funding": [ { @@ -808,7 +808,7 @@ "type": "tidelift" } ], - "time": "2022-10-25T13:49:28+00:00" + "time": "2022-10-26T14:07:24+00:00" }, { "name": "influxdb/influxdb-php", @@ -931,6 +931,72 @@ }, "time": "2021-02-04T16:20:16+00:00" }, + { + "name": "laravel/pint", + "version": "v1.2.0", + "source": { + "type": "git", + "url": "https://github.com/laravel/pint.git", + "reference": "1d276e4c803397a26cc337df908f55c2a4e90d86" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/laravel/pint/zipball/1d276e4c803397a26cc337df908f55c2a4e90d86", + "reference": "1d276e4c803397a26cc337df908f55c2a4e90d86", + "shasum": "" + }, + "require": { + "ext-json": "*", + "ext-mbstring": "*", + "ext-tokenizer": "*", + "ext-xml": "*", + "php": "^8.0" + }, + "require-dev": { + "friendsofphp/php-cs-fixer": "^3.11.0", + "illuminate/view": "^9.27", + "laravel-zero/framework": "^9.1.3", + "mockery/mockery": "^1.5.0", + "nunomaduro/larastan": "^2.2", + "nunomaduro/termwind": "^1.14.0", + "pestphp/pest": "^1.22.1" + }, + "bin": [ + "builds/pint" + ], + "type": "project", + "autoload": { + "psr-4": { + "App\\": "app/", + "Database\\Seeders\\": "database/seeders/", + "Database\\Factories\\": "database/factories/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Nuno Maduro", + "email": "enunomaduro@gmail.com" + } + ], + "description": "An opinionated code formatter for PHP.", + "homepage": "https://laravel.com", + "keywords": [ + "format", + "formatter", + "lint", + "linter", + "php" + ], + "support": { + "issues": "https://github.com/laravel/pint/issues", + "source": "https://github.com/laravel/pint" + }, + "time": "2022-09-13T15:07:15+00:00" + }, { "name": "matomo/device-detector", "version": "6.0.0", @@ -1658,16 +1724,16 @@ }, { "name": "symfony/polyfill-php80", - "version": "v1.26.0", + "version": "v1.27.0", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-php80.git", - "reference": "cfa0ae98841b9e461207c13ab093d76b0fa7bace" + "reference": "7a6ff3f1959bb01aefccb463a0f2cd3d3d2fd936" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-php80/zipball/cfa0ae98841b9e461207c13ab093d76b0fa7bace", - "reference": "cfa0ae98841b9e461207c13ab093d76b0fa7bace", + "url": "https://api.github.com/repos/symfony/polyfill-php80/zipball/7a6ff3f1959bb01aefccb463a0f2cd3d3d2fd936", + "reference": "7a6ff3f1959bb01aefccb463a0f2cd3d3d2fd936", "shasum": "" }, "require": { @@ -1676,7 +1742,7 @@ "type": "library", "extra": { "branch-alias": { - "dev-main": "1.26-dev" + "dev-main": "1.27-dev" }, "thanks": { "name": "symfony/polyfill", @@ -1721,7 +1787,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-php80/tree/v1.26.0" + "source": "https://github.com/symfony/polyfill-php80/tree/v1.27.0" }, "funding": [ { @@ -1737,7 +1803,7 @@ "type": "tidelift" } ], - "time": "2022-05-10T07:21:04+00:00" + "time": "2022-11-03T14:55:06+00:00" }, { "name": "utopia-php/abuse", @@ -2176,7 +2242,7 @@ "source": { "type": "git", "url": "https://github.com/utopia-php/dsn.git", - "reference": "9a524236c37474fcb121dc0d9eb13a0bcead9cf6" + "reference": "3a723fd6c52a161099145b5d96a535895de17069" }, "require": { "php": ">=8.0" @@ -2215,7 +2281,7 @@ "upf", "utopia" ], - "time": "2022-10-26T03:49:15+00:00" + "time": "2022-10-26T08:51:15+00:00" }, { "name": "utopia-php/framework", @@ -2708,23 +2774,25 @@ }, { "name": "utopia-php/system", - "version": "0.4.0", + "version": "0.6.0", "source": { "type": "git", "url": "https://github.com/utopia-php/system.git", - "reference": "67c92c66ce8f0cc925a00bca89f7a188bf9183c0" + "reference": "289c4327713deadc9c748b5317d248133a02f245" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/system/zipball/67c92c66ce8f0cc925a00bca89f7a188bf9183c0", - "reference": "67c92c66ce8f0cc925a00bca89f7a188bf9183c0", + "url": "https://api.github.com/repos/utopia-php/system/zipball/289c4327713deadc9c748b5317d248133a02f245", + "reference": "289c4327713deadc9c748b5317d248133a02f245", "shasum": "" }, "require": { + "laravel/pint": "1.2.*", "php": ">=7.4" }, "require-dev": { "phpunit/phpunit": "^9.3", + "squizlabs/php_codesniffer": "^3.6", "vimeo/psalm": "4.0.1" }, "type": "library", @@ -2757,9 +2825,9 @@ ], "support": { "issues": "https://github.com/utopia-php/system/issues", - "source": "https://github.com/utopia-php/system/tree/0.4.0" + "source": "https://github.com/utopia-php/system/tree/0.6.0" }, - "time": "2021-02-04T14:14:49+00:00" + "time": "2022-11-07T13:51:59+00:00" }, { "name": "utopia-php/websocket", @@ -3183,16 +3251,16 @@ }, { "name": "nikic/php-parser", - "version": "v4.15.1", + "version": "v4.15.2", "source": { "type": "git", "url": "https://github.com/nikic/PHP-Parser.git", - "reference": "0ef6c55a3f47f89d7a374e6f835197a0b5fcf900" + "reference": "f59bbe44bf7d96f24f3e2b4ddc21cd52c1d2adbc" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/0ef6c55a3f47f89d7a374e6f835197a0b5fcf900", - "reference": "0ef6c55a3f47f89d7a374e6f835197a0b5fcf900", + "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/f59bbe44bf7d96f24f3e2b4ddc21cd52c1d2adbc", + "reference": "f59bbe44bf7d96f24f3e2b4ddc21cd52c1d2adbc", "shasum": "" }, "require": { @@ -3233,9 +3301,9 @@ ], "support": { "issues": "https://github.com/nikic/PHP-Parser/issues", - "source": "https://github.com/nikic/PHP-Parser/tree/v4.15.1" + "source": "https://github.com/nikic/PHP-Parser/tree/v4.15.2" }, - "time": "2022-09-04T07:30:47+00:00" + "time": "2022-11-12T15:38:23+00:00" }, { "name": "phar-io/manifest", @@ -3582,16 +3650,16 @@ }, { "name": "phpunit/php-code-coverage", - "version": "9.2.17", + "version": "9.2.18", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/php-code-coverage.git", - "reference": "aa94dc41e8661fe90c7316849907cba3007b10d8" + "reference": "12fddc491826940cf9b7e88ad9664cf51f0f6d0a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/aa94dc41e8661fe90c7316849907cba3007b10d8", - "reference": "aa94dc41e8661fe90c7316849907cba3007b10d8", + "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/12fddc491826940cf9b7e88ad9664cf51f0f6d0a", + "reference": "12fddc491826940cf9b7e88ad9664cf51f0f6d0a", "shasum": "" }, "require": { @@ -3647,7 +3715,7 @@ ], "support": { "issues": "https://github.com/sebastianbergmann/php-code-coverage/issues", - "source": "https://github.com/sebastianbergmann/php-code-coverage/tree/9.2.17" + "source": "https://github.com/sebastianbergmann/php-code-coverage/tree/9.2.18" }, "funding": [ { @@ -3655,7 +3723,7 @@ "type": "github" } ], - "time": "2022-08-30T12:24:04+00:00" + "time": "2022-10-27T13:35:33+00:00" }, { "name": "phpunit/php-file-iterator", @@ -5065,16 +5133,16 @@ }, { "name": "symfony/polyfill-ctype", - "version": "v1.26.0", + "version": "v1.27.0", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-ctype.git", - "reference": "6fd1b9a79f6e3cf65f9e679b23af304cd9e010d4" + "reference": "5bbc823adecdae860bb64756d639ecfec17b050a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-ctype/zipball/6fd1b9a79f6e3cf65f9e679b23af304cd9e010d4", - "reference": "6fd1b9a79f6e3cf65f9e679b23af304cd9e010d4", + "url": "https://api.github.com/repos/symfony/polyfill-ctype/zipball/5bbc823adecdae860bb64756d639ecfec17b050a", + "reference": "5bbc823adecdae860bb64756d639ecfec17b050a", "shasum": "" }, "require": { @@ -5089,7 +5157,7 @@ "type": "library", "extra": { "branch-alias": { - "dev-main": "1.26-dev" + "dev-main": "1.27-dev" }, "thanks": { "name": "symfony/polyfill", @@ -5127,7 +5195,7 @@ "portable" ], "support": { - "source": "https://github.com/symfony/polyfill-ctype/tree/v1.26.0" + "source": "https://github.com/symfony/polyfill-ctype/tree/v1.27.0" }, "funding": [ { @@ -5143,20 +5211,20 @@ "type": "tidelift" } ], - "time": "2022-05-24T11:49:31+00:00" + "time": "2022-11-03T14:55:06+00:00" }, { "name": "symfony/polyfill-mbstring", - "version": "v1.26.0", + "version": "v1.27.0", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-mbstring.git", - "reference": "9344f9cb97f3b19424af1a21a3b0e75b0a7d8d7e" + "reference": "8ad114f6b39e2c98a8b0e3bd907732c207c2b534" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-mbstring/zipball/9344f9cb97f3b19424af1a21a3b0e75b0a7d8d7e", - "reference": "9344f9cb97f3b19424af1a21a3b0e75b0a7d8d7e", + "url": "https://api.github.com/repos/symfony/polyfill-mbstring/zipball/8ad114f6b39e2c98a8b0e3bd907732c207c2b534", + "reference": "8ad114f6b39e2c98a8b0e3bd907732c207c2b534", "shasum": "" }, "require": { @@ -5171,7 +5239,7 @@ "type": "library", "extra": { "branch-alias": { - "dev-main": "1.26-dev" + "dev-main": "1.27-dev" }, "thanks": { "name": "symfony/polyfill", @@ -5210,7 +5278,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-mbstring/tree/v1.26.0" + "source": "https://github.com/symfony/polyfill-mbstring/tree/v1.27.0" }, "funding": [ { @@ -5226,7 +5294,7 @@ "type": "tidelift" } ], - "time": "2022-05-24T11:49:31+00:00" + "time": "2022-11-03T14:55:06+00:00" }, { "name": "textalk/websocket", diff --git a/docker-compose.yml b/docker-compose.yml index 87bf5d5efb..6089f06055 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -131,7 +131,7 @@ services: - _APP_STORAGE_ANTIVIRUS - _APP_STORAGE_ANTIVIRUS_HOST - _APP_STORAGE_ANTIVIRUS_PORT - - _APP_STORAGE_CONNECTION + - _APP_CONNECTIONS_STORAGE - _APP_FUNCTIONS_SIZE_LIMIT - _APP_FUNCTIONS_TIMEOUT - _APP_FUNCTIONS_BUILD_TIMEOUT @@ -284,7 +284,7 @@ services: - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS - - _APP_STORAGE_CONNECTION + - _APP_CONNECTIONS_STORAGE - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG - _APP_EXECUTOR_SECRET @@ -452,7 +452,7 @@ services: - OPEN_RUNTIMES_NETWORK - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG - - _APP_STORAGE_CONNECTION + - _APP_CONNECTIONS_STORAGE - DOCKERHUB_PULL_USERNAME - DOCKERHUB_PULL_PASSWORD diff --git a/src/Appwrite/Resque/Worker.php b/src/Appwrite/Resque/Worker.php index c6e5e90c5e..0d0f982312 100644 --- a/src/Appwrite/Resque/Worker.php +++ b/src/Appwrite/Resque/Worker.php @@ -285,7 +285,7 @@ abstract class Worker */ public function getDevice($root): Device { - $connection = App::getEnv('_APP_STORAGE_CONNECTION', ''); + $connection = App::getEnv('_APP_CONNECTIONS_STORAGE', ''); try { $dsn = new DSN($connection); From 8b431a8e44d19f74d2cd4ea22b80f2a2162cba7b Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Thu, 17 Nov 2022 06:47:43 +0000 Subject: [PATCH 03/12] feat: remove executor --- app/executor.php | 793 ----------------------------------------------- 1 file changed, 793 deletions(-) delete mode 100644 app/executor.php diff --git a/app/executor.php b/app/executor.php deleted file mode 100644 index 68dc207da1..0000000000 --- a/app/executor.php +++ /dev/null @@ -1,793 +0,0 @@ -column('id', Swoole\Table::TYPE_STRING, 256); -$activeRuntimes->column('created', Swoole\Table::TYPE_INT, 8); -$activeRuntimes->column('updated', Swoole\Table::TYPE_INT, 8); -$activeRuntimes->column('name', Swoole\Table::TYPE_STRING, 128); -$activeRuntimes->column('status', Swoole\Table::TYPE_STRING, 128); -$activeRuntimes->column('key', Swoole\Table::TYPE_STRING, 256); -$activeRuntimes->create(); - -/** - * Create orchestration pool - */ -$orchestrationPool = new ConnectionPool(function () { - $dockerUser = App::getEnv('DOCKERHUB_PULL_USERNAME', null); - $dockerPass = App::getEnv('DOCKERHUB_PULL_PASSWORD', null); - $orchestration = new Orchestration(new DockerCLI($dockerUser, $dockerPass)); - return $orchestration; -}, 10); - - -/** - * Create logger instance - */ -$providerName = App::getEnv('_APP_LOGGING_PROVIDER', ''); -$providerConfig = App::getEnv('_APP_LOGGING_CONFIG', ''); -$logger = null; - -if (!empty($providerName) && !empty($providerConfig) && Logger::hasProvider($providerName)) { - $classname = '\\Utopia\\Logger\\Adapter\\' . \ucfirst($providerName); - $adapter = new $classname($providerConfig); - $logger = new Logger($adapter); -} - -function logError(Throwable $error, string $action, Utopia\Route $route = null) -{ - global $logger; - - if ($logger) { - $version = App::getEnv('_APP_VERSION', 'UNKNOWN'); - - $log = new Log(); - $log->setNamespace("executor"); - $log->setServer(\gethostname()); - $log->setVersion($version); - $log->setType(Log::TYPE_ERROR); - $log->setMessage($error->getMessage()); - - if ($route) { - $log->addTag('method', $route->getMethod()); - $log->addTag('url', $route->getPath()); - } - - $log->addTag('code', $error->getCode()); - $log->addTag('verboseType', get_class($error)); - - $log->addExtra('file', $error->getFile()); - $log->addExtra('line', $error->getLine()); - $log->addExtra('trace', $error->getTraceAsString()); - $log->addExtra('detailedTrace', $error->getTrace()); - - $log->setAction($action); - - $isProduction = App::getEnv('_APP_ENV', 'development') === 'production'; - $log->setEnvironment($isProduction ? Log::ENVIRONMENT_PRODUCTION : Log::ENVIRONMENT_STAGING); - - $responseCode = $logger->addLog($log); - Console::info('Executor log pushed with status code: ' . $responseCode); - } - - Console::error('[Error] Type: ' . get_class($error)); - Console::error('[Error] Message: ' . $error->getMessage()); - Console::error('[Error] File: ' . $error->getFile()); - Console::error('[Error] Line: ' . $error->getLine()); -} - -function getStorageDevice($root): Device -{ - $connection = App::getEnv('_APP_CONNECTIONS_STORAGE', ''); - - try { - $dsn = new DSN($connection); - $device = $dsn->getScheme(); - $accessKey = $dsn->getUser(); - $accessSecret = $dsn->getPassword(); - $bucket = $dsn->getPath(); - $region = $dsn->getParam('region'); - $acl = 'private'; - } catch (\Exception $e) { - Console::error($e->getMessage() . 'Defaulting to Local storage.'); - $device = 'Local'; - } - - switch ($device) { - case Storage::DEVICE_LOCAL: - default: - return new Local($root); - case Storage::DEVICE_S3: - return new S3($root, $accessKey, $accessSecret, $bucket, $region, $acl); - case Storage::DEVICE_DO_SPACES: - return new DOSpaces($root, $accessKey, $accessSecret, $bucket, $region, $acl); - case Storage::DEVICE_BACKBLAZE: - return new Backblaze($root, $accessKey, $accessSecret, $bucket, $region, $acl); - case Storage::DEVICE_LINODE: - return new Linode($root, $accessKey, $accessSecret, $bucket, $region, $acl); - case Storage::DEVICE_WASABI: - return new Wasabi($root, $accessKey, $accessSecret, $bucket, $region, $acl); - } -} - -App::post('/v1/runtimes') - ->desc("Create a new runtime server") - ->param('runtimeId', '', new Text(64), 'Unique runtime ID.') - ->param('source', '', new Text(0), 'Path to source files.') - ->param('destination', '', new Text(0), 'Destination folder to store build files into.', true) - ->param('vars', [], new Assoc(), 'Environment Variables required for the build.') - ->param('commands', [], new ArrayList(new Text(1024), 100), 'Commands required to build the container. Maximum of 100 commands are allowed, each 1024 characters long.') - ->param('runtime', '', new Text(128), 'Runtime for the cloud function.') - ->param('baseImage', '', new Text(128), 'Base image name of the runtime.') - ->param('entrypoint', '', new Text(256), 'Entrypoint of the code file.', true) - ->param('remove', false, new Boolean(), 'Remove a runtime after execution.') - ->param('workdir', '', new Text(256), 'Working directory.', true) - ->inject('orchestrationPool') - ->inject('activeRuntimes') - ->inject('response') - ->action(function (string $runtimeId, string $source, string $destination, array $vars, array $commands, string $runtime, string $baseImage, string $entrypoint, bool $remove, string $workdir, $orchestrationPool, $activeRuntimes, Response $response) { - if ($activeRuntimes->exists($runtimeId)) { - if ($activeRuntimes->get($runtimeId)['status'] == 'pending') { - throw new \Exception('A runtime with the same ID is already being created. Attempt a execution soon.', 500); - } - - throw new Exception('Runtime already exists.', 409); - } - - $container = []; - $containerId = ''; - $stdout = ''; - $stderr = ''; - $startTime = DateTime::now(); - $startTimeUnix = (new \DateTime($startTime))->getTimestamp(); - $endTimeUnix = 0; - $orchestration = $orchestrationPool->get(); - - $secret = \bin2hex(\random_bytes(16)); - - if (!$remove) { - $activeRuntimes->set($runtimeId, [ - 'id' => $containerId, - 'name' => $runtimeId, - 'created' => $startTimeUnix, - 'updated' => $endTimeUnix, - 'status' => 'pending', - 'key' => $secret, - ]); - } - - try { - Console::info('Building container : ' . $runtimeId); - - /** - * Temporary file paths in the executor - */ - $tmpSource = "/tmp/$runtimeId/src/code.tar.gz"; - $tmpBuild = "/tmp/$runtimeId/builds/code.tar.gz"; - - /** - * Copy code files from source to a temporary location on the executor - */ - $sourceDevice = getStorageDevice("/"); - $localDevice = new Local(); - $buffer = $sourceDevice->read($source); - if (!$localDevice->write($tmpSource, $buffer)) { - throw new Exception('Failed to copy source code to temporary directory', 500); - }; - - /** - * Create the mount folder - */ - if (!\file_exists(\dirname($tmpBuild))) { - if (!@\mkdir(\dirname($tmpBuild), 0755, true)) { - throw new Exception("Failed to create temporary directory", 500); - } - } - - /** - * Create container - */ - $vars = \array_merge($vars, [ - 'INTERNAL_RUNTIME_KEY' => $secret, - 'INTERNAL_RUNTIME_ENTRYPOINT' => $entrypoint, - ]); - $vars = array_map(fn ($v) => strval($v), $vars); - $orchestration - ->setCpus((int) App::getEnv('_APP_FUNCTIONS_CPUS', 0)) - ->setMemory((int) App::getEnv('_APP_FUNCTIONS_MEMORY', 0)) - ->setSwap((int) App::getEnv('_APP_FUNCTIONS_MEMORY_SWAP', 0)); - - /** Keep the container alive if we have commands to be executed */ - $entrypoint = !empty($commands) ? [ - 'tail', - '-f', - '/dev/null' - ] : []; - - $containerId = $orchestration->run( - image: $baseImage, - name: $runtimeId, - hostname: $runtimeId, - vars: $vars, - command: $entrypoint, - labels: [ - 'openruntimes-id' => $runtimeId, - 'openruntimes-type' => 'runtime', - 'openruntimes-created' => strval($startTimeUnix), - 'openruntimes-runtime' => $runtime, - ], - workdir: $workdir, - volumes: [ - \dirname($tmpSource) . ':/tmp:rw', - \dirname($tmpBuild) . ':/usr/code:rw' - ] - ); - - if (empty($containerId)) { - throw new Exception('Failed to create build container', 500); - } - - $orchestration->networkConnect($runtimeId, App::getEnv('OPEN_RUNTIMES_NETWORK', 'appwrite_runtimes')); - - /** - * Execute any commands if they were provided - */ - if (!empty($commands)) { - $status = $orchestration->execute( - name: $runtimeId, - command: $commands, - stdout: $stdout, - stderr: $stderr, - timeout: App::getEnv('_APP_FUNCTIONS_BUILD_TIMEOUT', 900) - ); - - if (!$status) { - throw new Exception('Failed to build dependenices ' . $stderr, 500); - } - } - - /** - * Move built code to expected build directory - */ - if (!empty($destination)) { - // Check if the build was successful by checking if file exists - if (!\file_exists($tmpBuild)) { - throw new Exception('Something went wrong during the build process', 500); - } - - $destinationDevice = getStorageDevice($destination); - $outputPath = $destinationDevice->getPath(\uniqid() . '.' . \pathinfo('code.tar.gz', PATHINFO_EXTENSION)); - - $buffer = $localDevice->read($tmpBuild); - if (!$destinationDevice->write($outputPath, $buffer, $localDevice->getFileMimeType($tmpBuild))) { - throw new Exception('Failed to move built code to storage', 500); - }; - - $container['outputPath'] = $outputPath; - } - - if (empty($stdout)) { - $stdout = 'Build Successful!'; - } - - $endTime = DateTime::now(); - $endTimeUnix = (new \DateTime($endTime))->getTimestamp(); - $duration = $endTimeUnix - $startTimeUnix; - - $container = array_merge($container, [ - 'status' => 'ready', - 'response' => \mb_strcut($stdout, 0, 1000000), // Limit to 1MB - 'stderr' => \mb_strcut($stderr, 0, 1000000), // Limit to 1MB - 'startTime' => $startTime, - 'endTime' => $endTime, - 'duration' => $duration, - ]); - - - if (!$remove) { - $activeRuntimes->set($runtimeId, [ - 'id' => $containerId, - 'name' => $runtimeId, - 'created' => $startTimeUnix, - 'updated' => $endTimeUnix, - 'status' => 'Up ' . \round($duration, 2) . 's', - 'key' => $secret, - ]); - } - - Console::success('Build Stage completed in ' . ($duration) . ' seconds'); - } catch (Throwable $th) { - Console::error('Build failed: ' . $th->getMessage() . $stdout); - - throw new Exception($th->getMessage() . $stdout, 500); - } finally { - // Container cleanup - if ($remove) { - if (!empty($containerId)) { - // If container properly created - $orchestration->remove($containerId, true); - $activeRuntimes->del($runtimeId); - } else { - // If whole creation failed, but container might have been initialized - try { - // Try to remove with contaier name instead of ID - $orchestration->remove($runtimeId, true); - $activeRuntimes->del($runtimeId); - } catch (Throwable $th) { - // If fails, means initialization also failed. - // Contianer is not there, no need to remove - } - } - } - - // Release orchestration back to pool, we are done with it - $orchestrationPool->put($orchestration); - } - - $response - ->setStatusCode(Response::STATUS_CODE_CREATED) - ->json($container); - }); - - -App::get('/v1/runtimes') - ->desc("List currently active runtimes") - ->inject('activeRuntimes') - ->inject('response') - ->action(function ($activeRuntimes, Response $response) { - $runtimes = []; - - foreach ($activeRuntimes as $runtime) { - $runtimes[] = $runtime; - } - - $response - ->setStatusCode(Response::STATUS_CODE_OK) - ->json($runtimes); - }); - -App::get('/v1/runtimes/:runtimeId') - ->desc("Get a runtime by its ID") - ->param('runtimeId', '', new Text(64), 'Runtime unique ID.') - ->inject('activeRuntimes') - ->inject('response') - ->action(function ($runtimeId, $activeRuntimes, Response $response) { - - if (!$activeRuntimes->exists($runtimeId)) { - throw new Exception('Runtime not found', 404); - } - - $runtime = $activeRuntimes->get($runtimeId); - - $response - ->setStatusCode(Response::STATUS_CODE_OK) - ->json($runtime); - }); - -App::delete('/v1/runtimes/:runtimeId') - ->desc('Delete a runtime') - ->param('runtimeId', '', new Text(64), 'Runtime unique ID.', false) - ->inject('orchestrationPool') - ->inject('activeRuntimes') - ->inject('response') - ->action(function (string $runtimeId, $orchestrationPool, $activeRuntimes, Response $response) { - - if (!$activeRuntimes->exists($runtimeId)) { - throw new Exception('Runtime not found', 404); - } - - Console::info('Deleting runtime: ' . $runtimeId); - - try { - $orchestration = $orchestrationPool->get(); - $orchestration->remove($runtimeId, true); - $activeRuntimes->del($runtimeId); - Console::success('Removed runtime container: ' . $runtimeId); - } finally { - $orchestrationPool->put($orchestration); - } - - // Remove all the build containers with that same ID - // TODO:: Delete build containers - // foreach ($buildIds as $buildId) { - // try { - // Console::info('Deleting build container : ' . $buildId); - // $status = $orchestration->remove('build-' . $buildId, true); - // } catch (Throwable $th) { - // Console::error($th->getMessage()); - // } - // } - - $response - ->setStatusCode(Response::STATUS_CODE_OK) - ->send(); - }); - - -App::post('/v1/execution') - ->desc('Create an execution') - ->param('runtimeId', '', new Text(64), 'The runtimeID to execute.') - ->param('vars', [], new Assoc(), 'Environment variables required for the build.') - ->param('data', '', new Text(8192), 'Data to be forwarded to the function, this is user specified.', true) - ->param('timeout', 15, new Range(1, (int) App::getEnv('_APP_FUNCTIONS_TIMEOUT', 900)), 'Function maximum execution time in seconds.') - ->inject('activeRuntimes') - ->inject('response') - ->action( - function (string $runtimeId, array $vars, string $data, $timeout, $activeRuntimes, Response $response) { - if (!$activeRuntimes->exists($runtimeId)) { - throw new Exception('Runtime not found. Please create the runtime.', 404); - } - - for ($i = 0; $i < 5; $i++) { - if ($activeRuntimes->get($runtimeId)['status'] === 'pending') { - Console::info('Waiting for runtime to be ready...'); - sleep(1); - } else { - break; - } - - if ($i === 4) { - throw new Exception('Runtime failed to launch in allocated time.', 500); - } - } - - $runtime = $activeRuntimes->get($runtimeId); - $secret = $runtime['key']; - if (empty($secret)) { - throw new Exception('Runtime secret not found. Please re-create the runtime.', 500); - } - - Console::info('Executing Runtime: ' . $runtimeId); - - $execution = []; - $executionStart = \microtime(true); - $stdout = ''; - $stderr = ''; - $res = ''; - $statusCode = 0; - $errNo = -1; - $executorResponse = ''; - - $timeout ??= (int) App::getEnv('_APP_FUNCTIONS_TIMEOUT', 900); - - $ch = \curl_init(); - $body = \json_encode([ - 'variables' => $vars, - 'payload' => $data, - 'timeout' => $timeout - ]); - \curl_setopt($ch, CURLOPT_URL, "http://" . $runtimeId . ":3000/"); - \curl_setopt($ch, CURLOPT_POST, true); - \curl_setopt($ch, CURLOPT_POSTFIELDS, $body); - \curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); - \curl_setopt($ch, CURLOPT_TIMEOUT, $timeout); - \curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 10); - - \curl_setopt($ch, CURLOPT_HTTPHEADER, [ - 'Content-Type: application/json', - 'Content-Length: ' . \strlen($body), - 'x-internal-challenge: ' . $secret, - 'host: null' - ]); - - $executorResponse = \curl_exec($ch); - $executorResponse = json_decode($executorResponse, true); - - $statusCode = \curl_getinfo($ch, CURLINFO_HTTP_CODE); - - $error = \curl_error($ch); - - $errNo = \curl_errno($ch); - - \curl_close($ch); - - switch (true) { - /** No Error. */ - case $errNo === 0: - break; - /** Runtime not ready for requests yet. 111 is the swoole error code for Connection Refused - see https://openswoole.com/docs/swoole-error-code */ - case $errNo === 111: - throw new Exception('An internal curl error has occurred within the executor! Error Msg: ' . $error, 406); - /** Any other CURL error */ - default: - throw new Exception('An internal curl error has occurred within the executor! Error Msg: ' . $error, 500); - } - - switch (true) { - case $statusCode >= 500: - $stderr = ($executorResponse ?? [])['stderr'] ?? 'Internal Runtime error.'; - $stdout = ($executorResponse ?? [])['stdout'] ?? 'Internal Runtime error.'; - break; - case $statusCode >= 100: - $stdout = $executorResponse['stdout']; - $res = $executorResponse['response']; - if (is_array($res)) { - $res = json_encode($res, JSON_UNESCAPED_UNICODE); - } - break; - default: - $stderr = ($executorResponse ?? [])['stderr'] ?? 'Execution failed.'; - $stdout = ($executorResponse ?? [])['stdout'] ?? ''; - break; - } - - $executionEnd = \microtime(true); - $executionTime = ($executionEnd - $executionStart); - $functionStatus = ($statusCode >= 500) ? 'failed' : 'completed'; - - Console::success('Function executed in ' . $executionTime . ' seconds, status: ' . $functionStatus); - - $execution = [ - 'status' => $functionStatus, - 'statusCode' => $statusCode, - 'response' => \mb_strcut($res, 0, 1000000), // Limit to 1MB - 'stdout' => \mb_strcut($stdout, 0, 1000000), // Limit to 1MB - 'stderr' => \mb_strcut($stderr, 0, 1000000), // Limit to 1MB - 'duration' => $executionTime, - ]; - - /** Update swoole table */ - $runtime['updated'] = \time(); - $activeRuntimes->set($runtimeId, $runtime); - - $response - ->setStatusCode(Response::STATUS_CODE_OK) - ->json($execution); - } - ); - -App::setMode(App::MODE_TYPE_PRODUCTION); // Define Mode - -$http = new Server("0.0.0.0", 80); - -/** Set Resources */ -App::setResource('orchestrationPool', fn() => $orchestrationPool); -App::setResource('activeRuntimes', fn() => $activeRuntimes); - -/** Set callbacks */ -App::error() - ->inject('utopia') - ->inject('error') - ->inject('request') - ->inject('response') - ->action(function (App $utopia, throwable $error, Request $request, Response $response) { - $route = $utopia->match($request); - logError($error, "httpError", $route); - - switch ($error->getCode()) { - case 400: // Error allowed publicly - case 401: // Error allowed publicly - case 402: // Error allowed publicly - case 403: // Error allowed publicly - case 404: // Error allowed publicly - case 406: // Error allowed publicly - case 409: // Error allowed publicly - case 412: // Error allowed publicly - case 425: // Error allowed publicly - case 429: // Error allowed publicly - case 501: // Error allowed publicly - case 503: // Error allowed publicly - $code = $error->getCode(); - break; - default: - $code = 500; // All other errors get the generic 500 server error status code - } - - $output = [ - 'message' => $error->getMessage(), - 'code' => $error->getCode(), - 'file' => $error->getFile(), - 'line' => $error->getLine(), - 'trace' => $error->getTrace(), - 'version' => App::getEnv('_APP_VERSION', 'UNKNOWN') - ]; - - $response - ->addHeader('Cache-Control', 'no-cache, no-store, must-revalidate') - ->addHeader('Expires', '0') - ->addHeader('Pragma', 'no-cache') - ->setStatusCode($code); - - $response->json($output); - }); - -App::init() - ->inject('request') - ->action(function (Request $request) { - $secretKey = $request->getHeader('x-appwrite-executor-key', ''); - if (empty($secretKey)) { - throw new Exception('Missing executor key', 401); - } - - if ($secretKey !== App::getEnv('_APP_EXECUTOR_SECRET', '')) { - throw new Exception('Missing executor key', 401); - } - }); - - -$http->on('start', function ($http) { - global $orchestrationPool; - global $activeRuntimes; - - /** - * Warmup: make sure images are ready to run fast 🚀 - */ - $runtimes = new Runtimes('v2'); - $allowList = empty(App::getEnv('_APP_FUNCTIONS_RUNTIMES')) ? [] : \explode(',', App::getEnv('_APP_FUNCTIONS_RUNTIMES')); - $runtimes = $runtimes->getAll(true, $allowList); - foreach ($runtimes as $runtime) { - go(function () use ($runtime, $orchestrationPool) { - try { - $orchestration = $orchestrationPool->get(); - Console::info('Warming up ' . $runtime['name'] . ' ' . $runtime['version'] . ' environment...'); - $response = $orchestration->pull($runtime['image']); - if ($response) { - Console::success("Successfully Warmed up {$runtime['name']} {$runtime['version']}!"); - } else { - Console::warning("Failed to Warmup {$runtime['name']} {$runtime['version']}!"); - } - } catch (\Throwable $th) { - } finally { - $orchestrationPool->put($orchestration); - } - }); - } - - /** - * Remove residual runtimes - */ - Console::info('Removing orphan runtimes...'); - try { - $orchestration = $orchestrationPool->get(); - $orphans = $orchestration->list(['label' => 'openruntimes-type=runtime']); - } finally { - $orchestrationPool->put($orchestration); - } - - foreach ($orphans as $runtime) { - go(function () use ($runtime, $orchestrationPool) { - try { - $orchestration = $orchestrationPool->get(); - $orchestration->remove($runtime->getName(), true); - Console::success("Successfully removed {$runtime->getName()}"); - } catch (\Throwable $th) { - Console::error('Orphan runtime deletion failed: ' . $th->getMessage()); - } finally { - $orchestrationPool->put($orchestration); - } - }); - } - - /** - * Register handlers for shutdown - */ - @Process::signal(SIGINT, function () use ($http) { - $http->shutdown(); - }); - - @Process::signal(SIGQUIT, function () use ($http) { - $http->shutdown(); - }); - - @Process::signal(SIGKILL, function () use ($http) { - $http->shutdown(); - }); - - @Process::signal(SIGTERM, function () use ($http) { - $http->shutdown(); - }); - - /** - * Run a maintenance worker every MAINTENANCE_INTERVAL seconds to remove inactive runtimes - */ - Timer::tick(MAINTENANCE_INTERVAL * 1000, function () use ($orchestrationPool, $activeRuntimes) { - Console::warning("Running maintenance task ..."); - foreach ($activeRuntimes as $runtime) { - $inactiveThreshold = \time() - App::getEnv('_APP_FUNCTIONS_INACTIVE_THRESHOLD', 60); - if ($runtime['updated'] < $inactiveThreshold) { - go(function () use ($runtime, $orchestrationPool, $activeRuntimes) { - try { - $orchestration = $orchestrationPool->get(); - $orchestration->remove($runtime['name'], true); - $activeRuntimes->del($runtime['name']); - Console::success("Successfully removed {$runtime['name']}"); - } catch (\Throwable $th) { - Console::error('Inactive Runtime deletion failed: ' . $th->getMessage()); - } finally { - $orchestrationPool->put($orchestration); - } - }); - } - } - }); -}); - - -$http->on('beforeShutdown', function () { - global $orchestrationPool; - Console::info('Cleaning up containers before shutdown...'); - - $orchestration = $orchestrationPool->get(); - $functionsToRemove = $orchestration->list(['label' => 'openruntimes-type=runtime']); - $orchestrationPool->put($orchestration); - - foreach ($functionsToRemove as $container) { - go(function () use ($orchestrationPool, $container) { - try { - $orchestration = $orchestrationPool->get(); - $orchestration->remove($container->getId(), true); - Console::info('Removed container ' . $container->getName()); - } catch (\Throwable $th) { - Console::error('Failed to remove container: ' . $container->getName()); - } finally { - $orchestrationPool->put($orchestration); - } - }); - } -}); - - -$http->on('request', function (SwooleRequest $swooleRequest, SwooleResponse $swooleResponse) { - $request = new Request($swooleRequest); - $response = new Response($swooleResponse); - $app = new App('UTC'); - - try { - $app->run($request, $response); - } catch (\Throwable $th) { - logError($th, "serverError"); - $swooleResponse->setStatusCode(500); - $output = [ - 'message' => 'Error: ' . $th->getMessage(), - 'code' => 500, - 'file' => $th->getFile(), - 'line' => $th->getLine(), - 'trace' => $th->getTrace() - ]; - $swooleResponse->end(\json_encode($output)); - } -}); - -$http->start(); From d830f5c262a8a929c6858a539cd63a2dcb74f899 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Thu, 17 Nov 2022 07:01:11 +0000 Subject: [PATCH 04/12] feat: review comments --- app/config/variables.php | 56 +++++++++++++++++----------------- app/init.php | 39 +++++++++++++++-------- src/Appwrite/Resque/Worker.php | 24 +++++++++------ 3 files changed, 69 insertions(+), 50 deletions(-) diff --git a/app/config/variables.php b/app/config/variables.php index eba040ac2d..a25bbca881 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -507,14 +507,6 @@ return [ 'question' => '', 'filter' => '' ], - [ - 'name' => '_APP_STORAGE_DEVICE', // TODO : Should we keep it or deprecate it considering we can get the device from the scheme of the DSN - 'description' => 'Select default storage device. The default value is \'Local\'. List of supported adapters are \'Local\', \'S3\', \'DOSpaces\', \'Backblaze\', \'Linode\' and \'Wasabi\'.', - 'introduction' => '0.13.0', - 'default' => 'Local', - 'required' => false, - 'question' => '', - ], [ 'name' => '_APP_CONNECTIONS_STORAGE', 'description' => 'A DSN representing the storage device to connect to. The DSN takes the following format ://:@:/?region=. For example, for S3: \'s3://access_key:access_secret@host:port/bucket?region=us-east-1\'. To use the local filesystem, you can leave this variable empty.', @@ -523,9 +515,17 @@ return [ 'required' => false, 'question' => '', ], + [ + 'name' => '_APP_STORAGE_DEVICE', + 'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.', + 'introduction' => '0.13.0', + 'default' => 'Local', + 'required' => false, + 'question' => '', + ], [ 'name' => '_APP_STORAGE_S3_ACCESS_KEY', - 'description' => 'Deprecated since 1.1.0. AWS S3 storage access key. Required when the storage adapter is set to S3. You can get your access key from your AWS console', + 'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.', 'introduction' => '0.13.0', 'default' => '', 'required' => false, @@ -533,7 +533,7 @@ return [ ], [ 'name' => '_APP_STORAGE_S3_SECRET', - 'description' => 'AWS S3 storage secret key. Required when the storage adapter is set to S3. You can get your secret key from your AWS console.', + 'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.', 'introduction' => '0.13.0', 'default' => '', 'required' => false, @@ -541,7 +541,7 @@ return [ ], [ 'name' => '_APP_STORAGE_S3_REGION', - 'description' => 'AWS S3 storage region. Required when storage adapter is set to S3. You can find your region info for your bucket from AWS console.', + 'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.', 'introduction' => '0.13.0', 'default' => 'us-east-1', 'required' => false, @@ -549,7 +549,7 @@ return [ ], [ 'name' => '_APP_STORAGE_S3_BUCKET', - 'description' => 'AWS S3 storage bucket. Required when storage adapter is set to S3. You can create buckets in your AWS console.', + 'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.', 'introduction' => '0.13.0', 'default' => '', 'required' => false, @@ -557,7 +557,7 @@ return [ ], [ 'name' => '_APP_STORAGE_DO_SPACES_ACCESS_KEY', - 'description' => 'DigitalOcean spaces access key. Required when the storage adapter is set to DOSpaces. You can get your access key from your DigitalOcean console.', + 'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.', 'introduction' => '0.13.0', 'default' => '', 'required' => false, @@ -565,7 +565,7 @@ return [ ], [ 'name' => '_APP_STORAGE_DO_SPACES_SECRET', - 'description' => 'DigitalOcean spaces secret key. Required when the storage adapter is set to DOSpaces. You can get your secret key from your DigitalOcean console.', + 'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.', 'introduction' => '0.13.0', 'default' => '', 'required' => false, @@ -573,7 +573,7 @@ return [ ], [ 'name' => '_APP_STORAGE_DO_SPACES_REGION', - 'description' => 'DigitalOcean spaces region. Required when storage adapter is set to DOSpaces. You can find your region info for your space from DigitalOcean console.', + 'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.', 'introduction' => '0.13.0', 'default' => 'us-east-1', 'required' => false, @@ -581,7 +581,7 @@ return [ ], [ 'name' => '_APP_STORAGE_DO_SPACES_BUCKET', - 'description' => 'DigitalOcean spaces bucket. Required when storage adapter is set to DOSpaces. You can create spaces in your DigitalOcean console.', + 'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.', 'introduction' => '0.13.0', 'default' => '', 'required' => false, @@ -589,7 +589,7 @@ return [ ], [ 'name' => '_APP_STORAGE_BACKBLAZE_ACCESS_KEY', - 'description' => 'Backblaze access key. Required when the storage adapter is set to Backblaze. Your Backblaze keyID will be your access key. You can get your keyID from your Backblaze console.', + 'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.', 'introduction' => '0.14.2', 'default' => '', 'required' => false, @@ -597,7 +597,7 @@ return [ ], [ 'name' => '_APP_STORAGE_BACKBLAZE_SECRET', - 'description' => 'Backblaze secret key. Required when the storage adapter is set to Backblaze. Your Backblaze applicationKey will be your secret key. You can get your applicationKey from your Backblaze console.', + 'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.', 'introduction' => '0.14.2', 'default' => '', 'required' => false, @@ -605,7 +605,7 @@ return [ ], [ 'name' => '_APP_STORAGE_BACKBLAZE_REGION', - 'description' => 'Backblaze region. Required when storage adapter is set to Backblaze. You can find your region info from your Backblaze console.', + 'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.', 'introduction' => '0.14.2', 'default' => 'us-west-004', 'required' => false, @@ -613,7 +613,7 @@ return [ ], [ 'name' => '_APP_STORAGE_BACKBLAZE_BUCKET', - 'description' => 'Backblaze bucket. Required when storage adapter is set to Backblaze. You can create your bucket from your Backblaze console.', + 'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.', 'introduction' => '0.14.2', 'default' => '', 'required' => false, @@ -621,7 +621,7 @@ return [ ], [ 'name' => '_APP_STORAGE_LINODE_ACCESS_KEY', - 'description' => 'Linode object storage access key. Required when the storage adapter is set to Linode. You can get your access key from your Linode console.', + 'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.', 'introduction' => '0.14.2', 'default' => '', 'required' => false, @@ -629,7 +629,7 @@ return [ ], [ 'name' => '_APP_STORAGE_LINODE_SECRET', - 'description' => 'Linode object storage secret key. Required when the storage adapter is set to Linode. You can get your secret key from your Linode console.', + 'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.', 'introduction' => '0.14.2', 'default' => '', 'required' => false, @@ -637,7 +637,7 @@ return [ ], [ 'name' => '_APP_STORAGE_LINODE_REGION', - 'description' => 'Linode object storage region. Required when storage adapter is set to Linode. You can find your region info from your Linode console.', + 'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.', 'introduction' => '0.14.2', 'default' => 'eu-central-1', 'required' => false, @@ -645,7 +645,7 @@ return [ ], [ 'name' => '_APP_STORAGE_LINODE_BUCKET', - 'description' => 'Linode object storage bucket. Required when storage adapter is set to Linode. You can create buckets in your Linode console.', + 'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.', 'introduction' => '0.14.2', 'default' => '', 'required' => false, @@ -653,7 +653,7 @@ return [ ], [ 'name' => '_APP_STORAGE_WASABI_ACCESS_KEY', - 'description' => 'Wasabi access key. Required when the storage adapter is set to Wasabi. You can get your access key from your Wasabi console.', + 'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.', 'introduction' => '0.14.2', 'default' => '', 'required' => false, @@ -661,7 +661,7 @@ return [ ], [ 'name' => '_APP_STORAGE_WASABI_SECRET', - 'description' => 'Wasabi secret key. Required when the storage adapter is set to Wasabi. You can get your secret key from your Wasabi console.', + 'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.', 'introduction' => '0.14.2', 'default' => '', 'required' => false, @@ -669,7 +669,7 @@ return [ ], [ 'name' => '_APP_STORAGE_WASABI_REGION', - 'description' => 'Wasabi region. Required when storage adapter is set to Wasabi. You can find your region info from your Wasabi console.', + 'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.', 'introduction' => '0.14.2', 'default' => 'eu-central-1', 'required' => false, @@ -677,7 +677,7 @@ return [ ], [ 'name' => '_APP_STORAGE_WASABI_BUCKET', - 'description' => 'Wasabi bucket. Required when storage adapter is set to Wasabi. You can create buckets in your Wasabi console.', + 'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.', 'introduction' => '0.14.2', 'default' => '', 'required' => false, diff --git a/app/init.php b/app/init.php index f9159827fd..2f9f0e2399 100644 --- a/app/init.php +++ b/app/init.php @@ -159,6 +159,13 @@ const DELETE_TYPE_SCHEDULES = 'schedules'; const COMPRESSION_TYPE_NONE = 'none'; const COMPRESSION_TYPE_GZIP = 'gzip'; const COMPRESSION_TYPE_ZSTD = 'zstd'; +// Storage Device Types +const STORAGE_DEVICE_LOCAL = 'file'; +const STORAGE_DEVICE_S3 = 's3'; +const STORAGE_DEVICE_DO_SPACES = 'dospaces'; +const STORAGE_DEVICE_BACKBLAZE= 'backblaze'; +const STORAGE_DEVICE_LINODE = 'linode'; +const STORAGE_DEVICE_WASABI = 'wasabi'; // Mail Types const MAIL_TYPE_VERIFICATION = 'verification'; const MAIL_TYPE_MAGIC_SESSION = 'magicSession'; @@ -1100,6 +1107,13 @@ function getDevice($root): Device { $connection = App::getEnv('_APP_CONNECTIONS_STORAGE', ''); + $acl = 'private'; + $device = ''; + $accessKey = ''; + $accessSecret = ''; + $bucket = ''; + $region = ''; + try { $dsn = new DSN($connection); $device = $dsn->getScheme(); @@ -1107,26 +1121,25 @@ function getDevice($root): Device $accessSecret = $dsn->getPassword(); $bucket = $dsn->getPath(); $region = $dsn->getParam('region'); - $acl = 'private'; } catch (\Exception $e) { - Console::error($e->getMessage() . 'Defaulting to Local storage.'); + Console::error($e->getMessage() . 'Invalid DSN. Defaulting to Local storage.'); $device = 'Local'; } switch ($device) { - case Storage::DEVICE_LOCAL: + case STORAGE_DEVICE_S3: + return new S3($root, $accessKey, $accessSecret, $bucket, $region, $acl); + case STORAGE_DEVICE_DO_SPACES: + return new DOSpaces($root, $accessKey, $accessSecret, $bucket, $region, $acl); + case STORAGE_DEVICE_BACKBLAZE: + return new Backblaze($root, $accessKey, $accessSecret, $bucket, $region, $acl); + case STORAGE_DEVICE_LINODE: + return new Linode($root, $accessKey, $accessSecret, $bucket, $region, $acl); + case STORAGE_DEVICE_WASABI: + return new Wasabi($root, $accessKey, $accessSecret, $bucket, $region, $acl); + case STORAGE_DEVICE_LOCAL: default: return new Local($root); - case Storage::DEVICE_S3: - return new S3($root, $accessKey, $accessSecret, $bucket, $region, $acl); - case Storage::DEVICE_DO_SPACES: - return new DOSpaces($root, $accessKey, $accessSecret, $bucket, $region, $acl); - case Storage::DEVICE_BACKBLAZE: - return new Backblaze($root, $accessKey, $accessSecret, $bucket, $region, $acl); - case Storage::DEVICE_LINODE: - return new Linode($root, $accessKey, $accessSecret, $bucket, $region, $acl); - case Storage::DEVICE_WASABI: - return new Wasabi($root, $accessKey, $accessSecret, $bucket, $region, $acl); } } diff --git a/src/Appwrite/Resque/Worker.php b/src/Appwrite/Resque/Worker.php index 17ed3e43cf..d7bedb6f2a 100644 --- a/src/Appwrite/Resque/Worker.php +++ b/src/Appwrite/Resque/Worker.php @@ -279,6 +279,13 @@ abstract class Worker { $connection = App::getEnv('_APP_CONNECTIONS_STORAGE', ''); + $acl = 'private'; + $device = ''; + $accessKey = ''; + $accessSecret = ''; + $bucket = ''; + $region = ''; + try { $dsn = new DSN($connection); $device = $dsn->getScheme(); @@ -286,24 +293,23 @@ abstract class Worker $accessSecret = $dsn->getPassword(); $bucket = $dsn->getPath(); $region = $dsn->getParam('region'); - $acl = 'private'; } catch (\Exception $e) { - Console::error($e->getMessage() . 'Defaulting to Local storage.'); - $device = 'Local'; + Console::error($e->getMessage() . 'Invalid DSN. Defaulting to Local storage.'); + $device = 'file'; } switch ($device) { - case Storage::DEVICE_S3: + case STORAGE_DEVICE_S3: return new S3($root, $accessKey, $accessSecret, $bucket, $region, $acl); - case Storage::DEVICE_DO_SPACES: + case STORAGE_DEVICE_DO_SPACES: return new DOSpaces($root, $accessKey, $accessSecret, $bucket, $region, $acl); - case Storage::DEVICE_BACKBLAZE: + case STORAGE_DEVICE_BACKBLAZE: return new Backblaze($root, $accessKey, $accessSecret, $bucket, $region, $acl); - case Storage::DEVICE_LINODE: + case STORAGE_DEVICE_LINODE: return new Linode($root, $accessKey, $accessSecret, $bucket, $region, $acl); - case Storage::DEVICE_WASABI: + case STORAGE_DEVICE_WASABI: return new Wasabi($root, $accessKey, $accessSecret, $bucket, $region, $acl); - case Storage::DEVICE_LOCAL: + case STORAGE_DEVICE_LOCAL: default: return new Local($root); } From 836ab62285c9443364cffc0292b07dffdbe9f85e Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Thu, 17 Nov 2022 07:02:51 +0000 Subject: [PATCH 05/12] feat: review comments --- app/init.php | 4 ++-- src/Appwrite/Resque/Worker.php | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/app/init.php b/app/init.php index 2f9f0e2399..50c5ab1a18 100644 --- a/app/init.php +++ b/app/init.php @@ -1108,7 +1108,7 @@ function getDevice($root): Device $connection = App::getEnv('_APP_CONNECTIONS_STORAGE', ''); $acl = 'private'; - $device = ''; + $device = STORAGE_DEVICE_LOCAL; $accessKey = ''; $accessSecret = ''; $bucket = ''; @@ -1123,7 +1123,7 @@ function getDevice($root): Device $region = $dsn->getParam('region'); } catch (\Exception $e) { Console::error($e->getMessage() . 'Invalid DSN. Defaulting to Local storage.'); - $device = 'Local'; + $device = STORAGE_DEVICE_LOCAL; } switch ($device) { diff --git a/src/Appwrite/Resque/Worker.php b/src/Appwrite/Resque/Worker.php index d7bedb6f2a..8503e18c3c 100644 --- a/src/Appwrite/Resque/Worker.php +++ b/src/Appwrite/Resque/Worker.php @@ -280,7 +280,7 @@ abstract class Worker $connection = App::getEnv('_APP_CONNECTIONS_STORAGE', ''); $acl = 'private'; - $device = ''; + $device = STORAGE_DEVICE_LOCAL; $accessKey = ''; $accessSecret = ''; $bucket = ''; @@ -295,7 +295,7 @@ abstract class Worker $region = $dsn->getParam('region'); } catch (\Exception $e) { Console::error($e->getMessage() . 'Invalid DSN. Defaulting to Local storage.'); - $device = 'file'; + $device = STORAGE_DEVICE_LOCAL; } switch ($device) { From 28087d708db7489ac9d884d6fb84f14a4387952d Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Thu, 17 Nov 2022 07:03:59 +0000 Subject: [PATCH 06/12] feat: review comments --- app/config/variables.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/config/variables.php b/app/config/variables.php index a25bbca881..9d5a0dd2e8 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -509,7 +509,7 @@ return [ ], [ 'name' => '_APP_CONNECTIONS_STORAGE', - 'description' => 'A DSN representing the storage device to connect to. The DSN takes the following format ://:@:/?region=. For example, for S3: \'s3://access_key:access_secret@host:port/bucket?region=us-east-1\'. To use the local filesystem, you can leave this variable empty.', + 'description' => 'A DSN representing the storage device to connect to. The DSN takes the following format ://:@:/?region=. For example, for S3: \'s3://access_key:access_secret@host:port/bucket?region=us-east-1\'. To use the local filesystem, you can leave this variable empty. Available devices are file, s3, dospaces, linode, backblaze and wasabi.', 'introduction' => '1.1.0', 'default' => '', 'required' => false, From 999dfe5d6363c8e4edad7f31cfb34fb022a66df5 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Thu, 17 Nov 2022 07:06:13 +0000 Subject: [PATCH 07/12] feat: review comments --- app/init.php | 1 + 1 file changed, 1 insertion(+) diff --git a/app/init.php b/app/init.php index 50c5ab1a18..03e5e6de75 100644 --- a/app/init.php +++ b/app/init.php @@ -55,6 +55,7 @@ use Utopia\Locale\Locale; use Utopia\Registry\Registry; use Utopia\Storage\Device; use Utopia\Storage\Storage; +use Utopia\DSN\DSN; use Utopia\Storage\Device\Backblaze; use Utopia\Storage\Device\DOSpaces; use Utopia\Storage\Device\Local; From ad5bd3b32803af699157d336be590ad8fbd8a9dc Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Thu, 17 Nov 2022 07:07:43 +0000 Subject: [PATCH 08/12] feat: review comments --- composer.json | 4 ---- 1 file changed, 4 deletions(-) diff --git a/composer.json b/composer.json index 3aee0f9ba9..ffd9c6b6b9 100644 --- a/composer.json +++ b/composer.json @@ -77,10 +77,6 @@ { "url": "https://github.com/appwrite/runtimes.git", "type": "git" - }, - { - "url": "https://github.com/utopia-php/dsn.git", - "type": "git" } ], "require-dev": { From 4271e34b399940854544ad410cf7b432412c5f61 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Thu, 17 Nov 2022 07:44:54 +0000 Subject: [PATCH 09/12] feat: review comments --- app/controllers/api/storage.php | 12 ++++++++- app/init.php | 2 +- app/views/install/compose.phtml | 44 ++------------------------------- app/workers/builds.php | 11 ++++++++- 4 files changed, 24 insertions(+), 45 deletions(-) diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index 8134c1d419..8dbdcb28c6 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -494,7 +494,17 @@ App::post('/v1/storage/buckets/:bucketId/files') } if ($chunksUploaded === $chunks) { - if (App::getEnv('_APP_STORAGE_ANTIVIRUS') === 'enabled' && $bucket->getAttribute('antivirus', true) && $fileSize <= APP_LIMIT_ANTIVIRUS && App::getEnv('_APP_STORAGE_DEVICE', Storage::DEVICE_LOCAL) === Storage::DEVICE_LOCAL) { // TODO : fetch from DSN + + $connection = App::getEnv('_APP_CONNECTIONS_STORAGE', ''); + $device = STORAGE_DEVICE_LOCAL; + try { + $dsn = new DSN($connection); + $device = $dsn->getScheme(); + } catch (\Exception $e) { + $device = STORAGE_DEVICE_LOCAL; + } + + if (App::getEnv('_APP_STORAGE_ANTIVIRUS') === 'enabled' && $bucket->getAttribute('antivirus', true) && $fileSize <= APP_LIMIT_ANTIVIRUS && $device === STORAGE_DEVICE_LOCAL) { $antivirus = new Network( App::getEnv('_APP_STORAGE_ANTIVIRUS_HOST', 'clamav'), (int) App::getEnv('_APP_STORAGE_ANTIVIRUS_PORT', 3310) diff --git a/app/init.php b/app/init.php index 03e5e6de75..305f1009d3 100644 --- a/app/init.php +++ b/app/init.php @@ -1123,7 +1123,7 @@ function getDevice($root): Device $bucket = $dsn->getPath(); $region = $dsn->getParam('region'); } catch (\Exception $e) { - Console::error($e->getMessage() . 'Invalid DSN. Defaulting to Local storage.'); + Console::eor($e->getMessage() . 'Invalid DSN. Defaulting to Local storage.'); $device = STORAGE_DEVICE_LOCAL; } diff --git a/app/views/install/compose.phtml b/app/views/install/compose.phtml index bfd1ab1dbe..37cc2beec0 100644 --- a/app/views/install/compose.phtml +++ b/app/views/install/compose.phtml @@ -120,27 +120,7 @@ services: - _APP_STORAGE_ANTIVIRUS - _APP_STORAGE_ANTIVIRUS_HOST - _APP_STORAGE_ANTIVIRUS_PORT - - _APP_STORAGE_DEVICE - - _APP_STORAGE_S3_ACCESS_KEY - - _APP_STORAGE_S3_SECRET - - _APP_STORAGE_S3_REGION - - _APP_STORAGE_S3_BUCKET - - _APP_STORAGE_DO_SPACES_ACCESS_KEY - - _APP_STORAGE_DO_SPACES_SECRET - - _APP_STORAGE_DO_SPACES_REGION - - _APP_STORAGE_DO_SPACES_BUCKET - - _APP_STORAGE_BACKBLAZE_ACCESS_KEY - - _APP_STORAGE_BACKBLAZE_SECRET - - _APP_STORAGE_BACKBLAZE_REGION - - _APP_STORAGE_BACKBLAZE_BUCKET - - _APP_STORAGE_LINODE_ACCESS_KEY - - _APP_STORAGE_LINODE_SECRET - - _APP_STORAGE_LINODE_REGION - - _APP_STORAGE_LINODE_BUCKET - - _APP_STORAGE_WASABI_ACCESS_KEY - - _APP_STORAGE_WASABI_SECRET - - _APP_STORAGE_WASABI_REGION - - _APP_STORAGE_WASABI_BUCKET + - _APP_CONNECTIONS_STORAGE - _APP_FUNCTIONS_SIZE_LIMIT - _APP_FUNCTIONS_TIMEOUT - _APP_FUNCTIONS_BUILD_TIMEOUT @@ -284,27 +264,7 @@ services: - _APP_REDIS_PORT - _APP_REDIS_USER - _APP_REDIS_PASS - - _APP_STORAGE_DEVICE - - _APP_STORAGE_S3_ACCESS_KEY - - _APP_STORAGE_S3_SECRET - - _APP_STORAGE_S3_REGION - - _APP_STORAGE_S3_BUCKET - - _APP_STORAGE_DO_SPACES_ACCESS_KEY - - _APP_STORAGE_DO_SPACES_SECRET - - _APP_STORAGE_DO_SPACES_REGION - - _APP_STORAGE_DO_SPACES_BUCKET - - _APP_STORAGE_BACKBLAZE_ACCESS_KEY - - _APP_STORAGE_BACKBLAZE_SECRET - - _APP_STORAGE_BACKBLAZE_REGION - - _APP_STORAGE_BACKBLAZE_BUCKET - - _APP_STORAGE_LINODE_ACCESS_KEY - - _APP_STORAGE_LINODE_SECRET - - _APP_STORAGE_LINODE_REGION - - _APP_STORAGE_LINODE_BUCKET - - _APP_STORAGE_WASABI_ACCESS_KEY - - _APP_STORAGE_WASABI_SECRET - - _APP_STORAGE_WASABI_REGION - - _APP_STORAGE_WASABI_BUCKET + - _APP_CONNECTIONS_STORAGE - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG - _APP_EXECUTOR_SECRET diff --git a/app/workers/builds.php b/app/workers/builds.php index bd7b627065..cb553652d8 100644 --- a/app/workers/builds.php +++ b/app/workers/builds.php @@ -79,6 +79,15 @@ class BuildsV1 extends Worker throw new Exception('Runtime "' . $function->getAttribute('runtime', '') . '" is not supported'); } + $connection = App::getEnv('_APP_CONNECTIONS_STORAGE', ''); + $device = STORAGE_DEVICE_LOCAL; + try { + $dsn = new DSN($connection); + $device = $dsn->getScheme(); + } catch (\Exception $e) { + $device = STORAGE_DEVICE_LOCAL; + } + $buildId = $deployment->getAttribute('buildId', ''); $startTime = DateTime::now(); if (empty($buildId)) { @@ -92,7 +101,7 @@ class BuildsV1 extends Worker 'outputPath' => '', 'runtime' => $function->getAttribute('runtime'), 'source' => $deployment->getAttribute('path'), - 'sourceType' => App::getEnv('_APP_STORAGE_DEVICE', Storage::DEVICE_LOCAL), // TODO : fetch from DSN + 'sourceType' => $device, 'stdout' => '', 'stderr' => '', 'endTime' => null, From 4f31fb5dc8668ad940939a8a03f7457f51dd7d80 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Thu, 17 Nov 2022 07:46:00 +0000 Subject: [PATCH 10/12] feat: linter --- app/controllers/api/storage.php | 1 - app/init.php | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index 8dbdcb28c6..8937beb851 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -494,7 +494,6 @@ App::post('/v1/storage/buckets/:bucketId/files') } if ($chunksUploaded === $chunks) { - $connection = App::getEnv('_APP_CONNECTIONS_STORAGE', ''); $device = STORAGE_DEVICE_LOCAL; try { diff --git a/app/init.php b/app/init.php index 305f1009d3..8768103e84 100644 --- a/app/init.php +++ b/app/init.php @@ -164,7 +164,7 @@ const COMPRESSION_TYPE_ZSTD = 'zstd'; const STORAGE_DEVICE_LOCAL = 'file'; const STORAGE_DEVICE_S3 = 's3'; const STORAGE_DEVICE_DO_SPACES = 'dospaces'; -const STORAGE_DEVICE_BACKBLAZE= 'backblaze'; +const STORAGE_DEVICE_BACKBLAZE = 'backblaze'; const STORAGE_DEVICE_LINODE = 'linode'; const STORAGE_DEVICE_WASABI = 'wasabi'; // Mail Types From 99c6dfb4aec2ec962ae22dbd26755616685dd474 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Thu, 17 Nov 2022 07:52:18 +0000 Subject: [PATCH 11/12] feat: update composer --- composer.lock | 117 +------------------------------------------------- 1 file changed, 1 insertion(+), 116 deletions(-) diff --git a/composer.lock b/composer.lock index 1332c846d7..1862ac982c 100644 --- a/composer.lock +++ b/composer.lock @@ -869,72 +869,6 @@ }, "time": "2022-09-13T15:07:15+00:00" }, - { - "name": "laravel/pint", - "version": "v1.2.0", - "source": { - "type": "git", - "url": "https://github.com/laravel/pint.git", - "reference": "1d276e4c803397a26cc337df908f55c2a4e90d86" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/laravel/pint/zipball/1d276e4c803397a26cc337df908f55c2a4e90d86", - "reference": "1d276e4c803397a26cc337df908f55c2a4e90d86", - "shasum": "" - }, - "require": { - "ext-json": "*", - "ext-mbstring": "*", - "ext-tokenizer": "*", - "ext-xml": "*", - "php": "^8.0" - }, - "require-dev": { - "friendsofphp/php-cs-fixer": "^3.11.0", - "illuminate/view": "^9.27", - "laravel-zero/framework": "^9.1.3", - "mockery/mockery": "^1.5.0", - "nunomaduro/larastan": "^2.2", - "nunomaduro/termwind": "^1.14.0", - "pestphp/pest": "^1.22.1" - }, - "bin": [ - "builds/pint" - ], - "type": "project", - "autoload": { - "psr-4": { - "App\\": "app/", - "Database\\Seeders\\": "database/seeders/", - "Database\\Factories\\": "database/factories/" - } - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "authors": [ - { - "name": "Nuno Maduro", - "email": "enunomaduro@gmail.com" - } - ], - "description": "An opinionated code formatter for PHP.", - "homepage": "https://laravel.com", - "keywords": [ - "format", - "formatter", - "lint", - "linter", - "php" - ], - "support": { - "issues": "https://github.com/laravel/pint/issues", - "source": "https://github.com/laravel/pint" - }, - "time": "2022-09-13T15:07:15+00:00" - }, { "name": "matomo/device-detector", "version": "6.0.0", @@ -2009,53 +1943,6 @@ }, "time": "2020-02-23T07:40:02+00:00" }, - { - "name": "utopia-php/dsn", - "version": "dev-dev", - "source": { - "type": "git", - "url": "https://github.com/utopia-php/dsn.git", - "reference": "3a723fd6c52a161099145b5d96a535895de17069" - }, - "require": { - "php": ">=8.0" - }, - "require-dev": { - "laravel/pint": "1.2.*", - "phpunit/phpunit": "^9.3", - "squizlabs/php_codesniffer": "^3.6", - "vimeo/psalm": "4.0.1" - }, - "type": "library", - "autoload": { - "psr-4": { - "Utopia\\DSN\\": "src/DSN" - } - }, - "scripts": { - "lint": [ - "./vendor/bin/pint --test" - ], - "format": [ - "./vendor/bin/pint" - ], - "test": [ - "./vendor/bin/phpunit --configuration phpunit.xml tests" - ] - }, - "license": [ - "MIT" - ], - "description": "A simple library for parsing and managing Data Source Names ( DSNs )", - "keywords": [ - "dsn", - "framework", - "php", - "upf", - "utopia" - ], - "time": "2022-10-26T08:51:15+00:00" - }, { "name": "utopia-php/framework", "version": "0.25.0", @@ -5391,9 +5278,7 @@ ], "aliases": [], "minimum-stability": "stable", - "stability-flags": { - "utopia-php/dsn": 20 - }, + "stability-flags": [], "prefer-stable": false, "prefer-lowest": false, "platform": { From b989a35b62d25804ca482cd0bc5afa0877f0a802 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Thu, 17 Nov 2022 08:04:05 +0000 Subject: [PATCH 12/12] feat: add todos --- app/controllers/api/storage.php | 2 +- app/workers/builds.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index 8937beb851..e2a3c2de6b 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -494,7 +494,7 @@ App::post('/v1/storage/buckets/:bucketId/files') } if ($chunksUploaded === $chunks) { - $connection = App::getEnv('_APP_CONNECTIONS_STORAGE', ''); + $connection = App::getEnv('_APP_CONNECTIONS_STORAGE', ''); /** @TODO : move this to the registry or someplace else */ $device = STORAGE_DEVICE_LOCAL; try { $dsn = new DSN($connection); diff --git a/app/workers/builds.php b/app/workers/builds.php index cb553652d8..348794c743 100644 --- a/app/workers/builds.php +++ b/app/workers/builds.php @@ -79,7 +79,7 @@ class BuildsV1 extends Worker throw new Exception('Runtime "' . $function->getAttribute('runtime', '') . '" is not supported'); } - $connection = App::getEnv('_APP_CONNECTIONS_STORAGE', ''); + $connection = App::getEnv('_APP_CONNECTIONS_STORAGE', ''); /** @TODO : move this to the registry or someplace else */ $device = STORAGE_DEVICE_LOCAL; try { $dsn = new DSN($connection);