diff --git a/.env b/.env index 3656448256..58906ccd33 100644 --- a/.env +++ b/.env @@ -29,27 +29,7 @@ _APP_CONNECTIONS_DB_CONSOLE=db_fra1_01=mariadb://user:password@mariadb:3306/appw _APP_CONNECTIONS_CACHE=redis_fra1_01=redis://redis:6379 _APP_CONNECTIONS_QUEUE=redis_fra1_01=redis://redis:6379 _APP_CONNECTIONS_PUBSUB=redis_fra1_01=redis://redis:6379 -_APP_STORAGE_DEVICE=Local -_APP_STORAGE_S3_ACCESS_KEY= -_APP_STORAGE_S3_SECRET= -_APP_STORAGE_S3_REGION=us-east-1 -_APP_STORAGE_S3_BUCKET= -_APP_STORAGE_DO_SPACES_ACCESS_KEY= -_APP_STORAGE_DO_SPACES_SECRET= -_APP_STORAGE_DO_SPACES_REGION=us-east-1 -_APP_STORAGE_DO_SPACES_BUCKET= -_APP_STORAGE_BACKBLAZE_ACCESS_KEY= -_APP_STORAGE_BACKBLAZE_SECRET= -_APP_STORAGE_BACKBLAZE_REGION=us-west-004 -_APP_STORAGE_BACKBLAZE_BUCKET= -_APP_STORAGE_LINODE_ACCESS_KEY= -_APP_STORAGE_LINODE_SECRET= -_APP_STORAGE_LINODE_REGION=eu-central-1 -_APP_STORAGE_LINODE_BUCKET= -_APP_STORAGE_WASABI_ACCESS_KEY= -_APP_STORAGE_WASABI_SECRET= -_APP_STORAGE_WASABI_REGION=eu-central-1 -_APP_STORAGE_WASABI_BUCKET= +_APP_CONNECTIONS_STORAGE=file://localhost _APP_STORAGE_ANTIVIRUS=disabled _APP_STORAGE_ANTIVIRUS_HOST=clamav _APP_STORAGE_ANTIVIRUS_PORT=3310 @@ -73,7 +53,6 @@ _APP_FUNCTIONS_CPUS=1 _APP_FUNCTIONS_MEMORY=512 _APP_FUNCTIONS_INACTIVE_THRESHOLD=600 _APP_FUNCTIONS_RUNTIMES_NETWORK=openruntimes-runtimes -_APP_FUNCTIONS_CONNECTION_STORAGE=file://localhost _APP_EXECUTOR_SECRET=your-secret-key _APP_EXECUTOR_HOST=http://exc1/v1 _APP_FUNCTIONS_RUNTIMES= diff --git a/Dockerfile b/Dockerfile index 543f39cd62..acd980e4a3 100755 --- a/Dockerfile +++ b/Dockerfile @@ -205,27 +205,7 @@ ENV _APP_SERVER=swoole \ _APP_STORAGE_ANTIVIRUS=enabled \ _APP_STORAGE_ANTIVIRUS_HOST=clamav \ _APP_STORAGE_ANTIVIRUS_PORT=3310 \ - _APP_STORAGE_DEVICE=Local \ - _APP_STORAGE_S3_ACCESS_KEY= \ - _APP_STORAGE_S3_SECRET= \ - _APP_STORAGE_S3_REGION= \ - _APP_STORAGE_S3_BUCKET= \ - _APP_STORAGE_DO_SPACES_ACCESS_KEY= \ - _APP_STORAGE_DO_SPACES_SECRET= \ - _APP_STORAGE_DO_SPACES_REGION= \ - _APP_STORAGE_DO_SPACES_BUCKET= \ - _APP_STORAGE_BACKBLAZE_ACCESS_KEY= \ - _APP_STORAGE_BACKBLAZE_SECRET= \ - _APP_STORAGE_BACKBLAZE_REGION= \ - _APP_STORAGE_BACKBLAZE_BUCKET= \ - _APP_STORAGE_LINODE_ACCESS_KEY= \ - _APP_STORAGE_LINODE_SECRET= \ - _APP_STORAGE_LINODE_REGION= \ - _APP_STORAGE_LINODE_BUCKET= \ - _APP_STORAGE_WASABI_ACCESS_KEY= \ - _APP_STORAGE_WASABI_SECRET= \ - _APP_STORAGE_WASABI_REGION= \ - _APP_STORAGE_WASABI_BUCKET= \ + _APP_CONNECTIONS_STORAGE= \ _APP_REDIS_HOST=redis \ _APP_REDIS_PORT=6379 \ _APP_DB_HOST=mariadb \ diff --git a/app/config/variables.php b/app/config/variables.php index 937996ef05..9d5a0dd2e8 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -507,9 +507,17 @@ return [ 'question' => '', 'filter' => '' ], + [ + 'name' => '_APP_CONNECTIONS_STORAGE', + 'description' => 'A DSN representing the storage device to connect to. The DSN takes the following format ://:@:/?region=. For example, for S3: \'s3://access_key:access_secret@host:port/bucket?region=us-east-1\'. To use the local filesystem, you can leave this variable empty. Available devices are file, s3, dospaces, linode, backblaze and wasabi.', + 'introduction' => '1.1.0', + 'default' => '', + 'required' => false, + 'question' => '', + ], [ 'name' => '_APP_STORAGE_DEVICE', - 'description' => 'Select default storage device. The default value is \'Local\'. List of supported adapters are \'Local\', \'S3\', \'DOSpaces\', \'Backblaze\', \'Linode\' and \'Wasabi\'.', + 'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.', 'introduction' => '0.13.0', 'default' => 'Local', 'required' => false, @@ -517,7 +525,7 @@ return [ ], [ 'name' => '_APP_STORAGE_S3_ACCESS_KEY', - 'description' => 'AWS S3 storage access key. Required when the storage adapter is set to S3. You can get your access key from your AWS console', + 'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.', 'introduction' => '0.13.0', 'default' => '', 'required' => false, @@ -525,7 +533,7 @@ return [ ], [ 'name' => '_APP_STORAGE_S3_SECRET', - 'description' => 'AWS S3 storage secret key. Required when the storage adapter is set to S3. You can get your secret key from your AWS console.', + 'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.', 'introduction' => '0.13.0', 'default' => '', 'required' => false, @@ -533,7 +541,7 @@ return [ ], [ 'name' => '_APP_STORAGE_S3_REGION', - 'description' => 'AWS S3 storage region. Required when storage adapter is set to S3. You can find your region info for your bucket from AWS console.', + 'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.', 'introduction' => '0.13.0', 'default' => 'us-east-1', 'required' => false, @@ -541,7 +549,7 @@ return [ ], [ 'name' => '_APP_STORAGE_S3_BUCKET', - 'description' => 'AWS S3 storage bucket. Required when storage adapter is set to S3. You can create buckets in your AWS console.', + 'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.', 'introduction' => '0.13.0', 'default' => '', 'required' => false, @@ -549,7 +557,7 @@ return [ ], [ 'name' => '_APP_STORAGE_DO_SPACES_ACCESS_KEY', - 'description' => 'DigitalOcean spaces access key. Required when the storage adapter is set to DOSpaces. You can get your access key from your DigitalOcean console.', + 'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.', 'introduction' => '0.13.0', 'default' => '', 'required' => false, @@ -557,7 +565,7 @@ return [ ], [ 'name' => '_APP_STORAGE_DO_SPACES_SECRET', - 'description' => 'DigitalOcean spaces secret key. Required when the storage adapter is set to DOSpaces. You can get your secret key from your DigitalOcean console.', + 'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.', 'introduction' => '0.13.0', 'default' => '', 'required' => false, @@ -565,7 +573,7 @@ return [ ], [ 'name' => '_APP_STORAGE_DO_SPACES_REGION', - 'description' => 'DigitalOcean spaces region. Required when storage adapter is set to DOSpaces. You can find your region info for your space from DigitalOcean console.', + 'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.', 'introduction' => '0.13.0', 'default' => 'us-east-1', 'required' => false, @@ -573,7 +581,7 @@ return [ ], [ 'name' => '_APP_STORAGE_DO_SPACES_BUCKET', - 'description' => 'DigitalOcean spaces bucket. Required when storage adapter is set to DOSpaces. You can create spaces in your DigitalOcean console.', + 'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.', 'introduction' => '0.13.0', 'default' => '', 'required' => false, @@ -581,7 +589,7 @@ return [ ], [ 'name' => '_APP_STORAGE_BACKBLAZE_ACCESS_KEY', - 'description' => 'Backblaze access key. Required when the storage adapter is set to Backblaze. Your Backblaze keyID will be your access key. You can get your keyID from your Backblaze console.', + 'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.', 'introduction' => '0.14.2', 'default' => '', 'required' => false, @@ -589,7 +597,7 @@ return [ ], [ 'name' => '_APP_STORAGE_BACKBLAZE_SECRET', - 'description' => 'Backblaze secret key. Required when the storage adapter is set to Backblaze. Your Backblaze applicationKey will be your secret key. You can get your applicationKey from your Backblaze console.', + 'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.', 'introduction' => '0.14.2', 'default' => '', 'required' => false, @@ -597,7 +605,7 @@ return [ ], [ 'name' => '_APP_STORAGE_BACKBLAZE_REGION', - 'description' => 'Backblaze region. Required when storage adapter is set to Backblaze. You can find your region info from your Backblaze console.', + 'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.', 'introduction' => '0.14.2', 'default' => 'us-west-004', 'required' => false, @@ -605,7 +613,7 @@ return [ ], [ 'name' => '_APP_STORAGE_BACKBLAZE_BUCKET', - 'description' => 'Backblaze bucket. Required when storage adapter is set to Backblaze. You can create your bucket from your Backblaze console.', + 'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.', 'introduction' => '0.14.2', 'default' => '', 'required' => false, @@ -613,7 +621,7 @@ return [ ], [ 'name' => '_APP_STORAGE_LINODE_ACCESS_KEY', - 'description' => 'Linode object storage access key. Required when the storage adapter is set to Linode. You can get your access key from your Linode console.', + 'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.', 'introduction' => '0.14.2', 'default' => '', 'required' => false, @@ -621,7 +629,7 @@ return [ ], [ 'name' => '_APP_STORAGE_LINODE_SECRET', - 'description' => 'Linode object storage secret key. Required when the storage adapter is set to Linode. You can get your secret key from your Linode console.', + 'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.', 'introduction' => '0.14.2', 'default' => '', 'required' => false, @@ -629,7 +637,7 @@ return [ ], [ 'name' => '_APP_STORAGE_LINODE_REGION', - 'description' => 'Linode object storage region. Required when storage adapter is set to Linode. You can find your region info from your Linode console.', + 'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.', 'introduction' => '0.14.2', 'default' => 'eu-central-1', 'required' => false, @@ -637,7 +645,7 @@ return [ ], [ 'name' => '_APP_STORAGE_LINODE_BUCKET', - 'description' => 'Linode object storage bucket. Required when storage adapter is set to Linode. You can create buckets in your Linode console.', + 'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.', 'introduction' => '0.14.2', 'default' => '', 'required' => false, @@ -645,7 +653,7 @@ return [ ], [ 'name' => '_APP_STORAGE_WASABI_ACCESS_KEY', - 'description' => 'Wasabi access key. Required when the storage adapter is set to Wasabi. You can get your access key from your Wasabi console.', + 'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.', 'introduction' => '0.14.2', 'default' => '', 'required' => false, @@ -653,7 +661,7 @@ return [ ], [ 'name' => '_APP_STORAGE_WASABI_SECRET', - 'description' => 'Wasabi secret key. Required when the storage adapter is set to Wasabi. You can get your secret key from your Wasabi console.', + 'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.', 'introduction' => '0.14.2', 'default' => '', 'required' => false, @@ -661,7 +669,7 @@ return [ ], [ 'name' => '_APP_STORAGE_WASABI_REGION', - 'description' => 'Wasabi region. Required when storage adapter is set to Wasabi. You can find your region info from your Wasabi console.', + 'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.', 'introduction' => '0.14.2', 'default' => 'eu-central-1', 'required' => false, @@ -669,7 +677,7 @@ return [ ], [ 'name' => '_APP_STORAGE_WASABI_BUCKET', - 'description' => 'Wasabi bucket. Required when storage adapter is set to Wasabi. You can create buckets in your Wasabi console.', + 'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.', 'introduction' => '0.14.2', 'default' => '', 'required' => false, @@ -860,15 +868,6 @@ return [ 'required' => false, 'question' => '', 'filter' => '' - ], - [ - 'name' => '_APP_FUNCTIONS_CONNECTION_STORAGE', - 'description' => 'DSN record of storage driver where Open Runtimes Executor stores output of Appwrite Function Deployment builds.', - 'introduction' => '1.2.0', - 'default' => '', - 'required' => false, - 'question' => '', - 'filter' => '' ] ], ], diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index 530f174cd0..e2a3c2de6b 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -494,7 +494,16 @@ App::post('/v1/storage/buckets/:bucketId/files') } if ($chunksUploaded === $chunks) { - if (App::getEnv('_APP_STORAGE_ANTIVIRUS') === 'enabled' && $bucket->getAttribute('antivirus', true) && $fileSize <= APP_LIMIT_ANTIVIRUS && App::getEnv('_APP_STORAGE_DEVICE', Storage::DEVICE_LOCAL) === Storage::DEVICE_LOCAL) { + $connection = App::getEnv('_APP_CONNECTIONS_STORAGE', ''); /** @TODO : move this to the registry or someplace else */ + $device = STORAGE_DEVICE_LOCAL; + try { + $dsn = new DSN($connection); + $device = $dsn->getScheme(); + } catch (\Exception $e) { + $device = STORAGE_DEVICE_LOCAL; + } + + if (App::getEnv('_APP_STORAGE_ANTIVIRUS') === 'enabled' && $bucket->getAttribute('antivirus', true) && $fileSize <= APP_LIMIT_ANTIVIRUS && $device === STORAGE_DEVICE_LOCAL) { $antivirus = new Network( App::getEnv('_APP_STORAGE_ANTIVIRUS_HOST', 'clamav'), (int) App::getEnv('_APP_STORAGE_ANTIVIRUS_PORT', 3310) diff --git a/app/init.php b/app/init.php index 709d2ba95d..8768103e84 100644 --- a/app/init.php +++ b/app/init.php @@ -26,7 +26,6 @@ use Appwrite\SMS\Adapter\TextMagic; use Appwrite\SMS\Adapter\Twilio; use Appwrite\SMS\Adapter\Msg91; use Appwrite\SMS\Adapter\Vonage; -use Appwrite\DSN\DSN; use Appwrite\Event\Audit; use Appwrite\Event\Database as EventDatabase; use Appwrite\Event\Event; @@ -56,6 +55,7 @@ use Utopia\Locale\Locale; use Utopia\Registry\Registry; use Utopia\Storage\Device; use Utopia\Storage\Storage; +use Utopia\DSN\DSN; use Utopia\Storage\Device\Backblaze; use Utopia\Storage\Device\DOSpaces; use Utopia\Storage\Device\Local; @@ -160,6 +160,13 @@ const DELETE_TYPE_SCHEDULES = 'schedules'; const COMPRESSION_TYPE_NONE = 'none'; const COMPRESSION_TYPE_GZIP = 'gzip'; const COMPRESSION_TYPE_ZSTD = 'zstd'; +// Storage Device Types +const STORAGE_DEVICE_LOCAL = 'file'; +const STORAGE_DEVICE_S3 = 's3'; +const STORAGE_DEVICE_DO_SPACES = 'dospaces'; +const STORAGE_DEVICE_BACKBLAZE = 'backblaze'; +const STORAGE_DEVICE_LINODE = 'linode'; +const STORAGE_DEVICE_WASABI = 'wasabi'; // Mail Types const MAIL_TYPE_VERIFICATION = 'verification'; const MAIL_TYPE_MAGIC_SESSION = 'magicSession'; @@ -1099,45 +1106,41 @@ App::setResource('deviceBuilds', function ($project) { function getDevice($root): Device { - switch (App::getEnv('_APP_STORAGE_DEVICE', Storage::DEVICE_LOCAL)) { - case Storage::DEVICE_LOCAL: + $connection = App::getEnv('_APP_CONNECTIONS_STORAGE', ''); + + $acl = 'private'; + $device = STORAGE_DEVICE_LOCAL; + $accessKey = ''; + $accessSecret = ''; + $bucket = ''; + $region = ''; + + try { + $dsn = new DSN($connection); + $device = $dsn->getScheme(); + $accessKey = $dsn->getUser(); + $accessSecret = $dsn->getPassword(); + $bucket = $dsn->getPath(); + $region = $dsn->getParam('region'); + } catch (\Exception $e) { + Console::eor($e->getMessage() . 'Invalid DSN. Defaulting to Local storage.'); + $device = STORAGE_DEVICE_LOCAL; + } + + switch ($device) { + case STORAGE_DEVICE_S3: + return new S3($root, $accessKey, $accessSecret, $bucket, $region, $acl); + case STORAGE_DEVICE_DO_SPACES: + return new DOSpaces($root, $accessKey, $accessSecret, $bucket, $region, $acl); + case STORAGE_DEVICE_BACKBLAZE: + return new Backblaze($root, $accessKey, $accessSecret, $bucket, $region, $acl); + case STORAGE_DEVICE_LINODE: + return new Linode($root, $accessKey, $accessSecret, $bucket, $region, $acl); + case STORAGE_DEVICE_WASABI: + return new Wasabi($root, $accessKey, $accessSecret, $bucket, $region, $acl); + case STORAGE_DEVICE_LOCAL: default: return new Local($root); - case Storage::DEVICE_S3: - $s3AccessKey = App::getEnv('_APP_STORAGE_S3_ACCESS_KEY', ''); - $s3SecretKey = App::getEnv('_APP_STORAGE_S3_SECRET', ''); - $s3Region = App::getEnv('_APP_STORAGE_S3_REGION', ''); - $s3Bucket = App::getEnv('_APP_STORAGE_S3_BUCKET', ''); - $s3Acl = 'private'; - return new S3($root, $s3AccessKey, $s3SecretKey, $s3Bucket, $s3Region, $s3Acl); - case Storage::DEVICE_DO_SPACES: - $doSpacesAccessKey = App::getEnv('_APP_STORAGE_DO_SPACES_ACCESS_KEY', ''); - $doSpacesSecretKey = App::getEnv('_APP_STORAGE_DO_SPACES_SECRET', ''); - $doSpacesRegion = App::getEnv('_APP_STORAGE_DO_SPACES_REGION', ''); - $doSpacesBucket = App::getEnv('_APP_STORAGE_DO_SPACES_BUCKET', ''); - $doSpacesAcl = 'private'; - return new DOSpaces($root, $doSpacesAccessKey, $doSpacesSecretKey, $doSpacesBucket, $doSpacesRegion, $doSpacesAcl); - case Storage::DEVICE_BACKBLAZE: - $backblazeAccessKey = App::getEnv('_APP_STORAGE_BACKBLAZE_ACCESS_KEY', ''); - $backblazeSecretKey = App::getEnv('_APP_STORAGE_BACKBLAZE_SECRET', ''); - $backblazeRegion = App::getEnv('_APP_STORAGE_BACKBLAZE_REGION', ''); - $backblazeBucket = App::getEnv('_APP_STORAGE_BACKBLAZE_BUCKET', ''); - $backblazeAcl = 'private'; - return new Backblaze($root, $backblazeAccessKey, $backblazeSecretKey, $backblazeBucket, $backblazeRegion, $backblazeAcl); - case Storage::DEVICE_LINODE: - $linodeAccessKey = App::getEnv('_APP_STORAGE_LINODE_ACCESS_KEY', ''); - $linodeSecretKey = App::getEnv('_APP_STORAGE_LINODE_SECRET', ''); - $linodeRegion = App::getEnv('_APP_STORAGE_LINODE_REGION', ''); - $linodeBucket = App::getEnv('_APP_STORAGE_LINODE_BUCKET', ''); - $linodeAcl = 'private'; - return new Linode($root, $linodeAccessKey, $linodeSecretKey, $linodeBucket, $linodeRegion, $linodeAcl); - case Storage::DEVICE_WASABI: - $wasabiAccessKey = App::getEnv('_APP_STORAGE_WASABI_ACCESS_KEY', ''); - $wasabiSecretKey = App::getEnv('_APP_STORAGE_WASABI_SECRET', ''); - $wasabiRegion = App::getEnv('_APP_STORAGE_WASABI_REGION', ''); - $wasabiBucket = App::getEnv('_APP_STORAGE_WASABI_BUCKET', ''); - $wasabiAcl = 'private'; - return new Wasabi($root, $wasabiAccessKey, $wasabiSecretKey, $wasabiBucket, $wasabiRegion, $wasabiAcl); } } diff --git a/app/views/install/compose.phtml b/app/views/install/compose.phtml index eb18756464..37cc2beec0 100644 --- a/app/views/install/compose.phtml +++ b/app/views/install/compose.phtml @@ -120,27 +120,7 @@ services: - _APP_STORAGE_ANTIVIRUS - _APP_STORAGE_ANTIVIRUS_HOST - _APP_STORAGE_ANTIVIRUS_PORT - - _APP_STORAGE_DEVICE - - _APP_STORAGE_S3_ACCESS_KEY - - _APP_STORAGE_S3_SECRET - - _APP_STORAGE_S3_REGION - - _APP_STORAGE_S3_BUCKET - - _APP_STORAGE_DO_SPACES_ACCESS_KEY - - _APP_STORAGE_DO_SPACES_SECRET - - _APP_STORAGE_DO_SPACES_REGION - - _APP_STORAGE_DO_SPACES_BUCKET - - _APP_STORAGE_BACKBLAZE_ACCESS_KEY - - _APP_STORAGE_BACKBLAZE_SECRET - - _APP_STORAGE_BACKBLAZE_REGION - - _APP_STORAGE_BACKBLAZE_BUCKET - - _APP_STORAGE_LINODE_ACCESS_KEY - - _APP_STORAGE_LINODE_SECRET - - _APP_STORAGE_LINODE_REGION - - _APP_STORAGE_LINODE_BUCKET - - _APP_STORAGE_WASABI_ACCESS_KEY - - _APP_STORAGE_WASABI_SECRET - - _APP_STORAGE_WASABI_REGION - - _APP_STORAGE_WASABI_BUCKET + - _APP_CONNECTIONS_STORAGE - _APP_FUNCTIONS_SIZE_LIMIT - _APP_FUNCTIONS_TIMEOUT - _APP_FUNCTIONS_BUILD_TIMEOUT @@ -284,27 +264,7 @@ services: - _APP_REDIS_PORT - _APP_REDIS_USER - _APP_REDIS_PASS - - _APP_STORAGE_DEVICE - - _APP_STORAGE_S3_ACCESS_KEY - - _APP_STORAGE_S3_SECRET - - _APP_STORAGE_S3_REGION - - _APP_STORAGE_S3_BUCKET - - _APP_STORAGE_DO_SPACES_ACCESS_KEY - - _APP_STORAGE_DO_SPACES_SECRET - - _APP_STORAGE_DO_SPACES_REGION - - _APP_STORAGE_DO_SPACES_BUCKET - - _APP_STORAGE_BACKBLAZE_ACCESS_KEY - - _APP_STORAGE_BACKBLAZE_SECRET - - _APP_STORAGE_BACKBLAZE_REGION - - _APP_STORAGE_BACKBLAZE_BUCKET - - _APP_STORAGE_LINODE_ACCESS_KEY - - _APP_STORAGE_LINODE_SECRET - - _APP_STORAGE_LINODE_REGION - - _APP_STORAGE_LINODE_BUCKET - - _APP_STORAGE_WASABI_ACCESS_KEY - - _APP_STORAGE_WASABI_SECRET - - _APP_STORAGE_WASABI_REGION - - _APP_STORAGE_WASABI_BUCKET + - _APP_CONNECTIONS_STORAGE - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG - _APP_EXECUTOR_SECRET @@ -598,7 +558,7 @@ services: - openruntimes-functions:/storage/functions:rw - /tmp:/tmp:rw environment: - - OPR_EXECUTOR_CONNECTION_STORAGE=$_APP_FUNCTIONS_CONNECTION_STORAGE + - OPR_EXECUTOR_CONNECTION_STORAGE=$_APP_CONNECTIONS_STORAGE - OPR_EXECUTOR_INACTIVE_TRESHOLD=$_APP_FUNCTIONS_INACTIVE_THRESHOLD - OPR_EXECUTOR_NETWORK=$_APP_FUNCTIONS_RUNTIMES_NETWORK - OPR_EXECUTOR_DOCKER_HUB_USERNAME=$_APP_DOCKER_HUB_USERNAME diff --git a/app/workers/builds.php b/app/workers/builds.php index e22807d30e..348794c743 100644 --- a/app/workers/builds.php +++ b/app/workers/builds.php @@ -79,6 +79,15 @@ class BuildsV1 extends Worker throw new Exception('Runtime "' . $function->getAttribute('runtime', '') . '" is not supported'); } + $connection = App::getEnv('_APP_CONNECTIONS_STORAGE', ''); /** @TODO : move this to the registry or someplace else */ + $device = STORAGE_DEVICE_LOCAL; + try { + $dsn = new DSN($connection); + $device = $dsn->getScheme(); + } catch (\Exception $e) { + $device = STORAGE_DEVICE_LOCAL; + } + $buildId = $deployment->getAttribute('buildId', ''); $startTime = DateTime::now(); if (empty($buildId)) { @@ -92,7 +101,7 @@ class BuildsV1 extends Worker 'outputPath' => '', 'runtime' => $function->getAttribute('runtime'), 'source' => $deployment->getAttribute('path'), - 'sourceType' => App::getEnv('_APP_STORAGE_DEVICE', Storage::DEVICE_LOCAL), + 'sourceType' => $device, 'stdout' => '', 'stderr' => '', 'endTime' => null, diff --git a/docker-compose.yml b/docker-compose.yml index 32d0d42501..b98cd9ed53 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -146,27 +146,7 @@ services: - _APP_STORAGE_ANTIVIRUS - _APP_STORAGE_ANTIVIRUS_HOST - _APP_STORAGE_ANTIVIRUS_PORT - - _APP_STORAGE_DEVICE - - _APP_STORAGE_S3_ACCESS_KEY - - _APP_STORAGE_S3_SECRET - - _APP_STORAGE_S3_REGION - - _APP_STORAGE_S3_BUCKET - - _APP_STORAGE_DO_SPACES_ACCESS_KEY - - _APP_STORAGE_DO_SPACES_SECRET - - _APP_STORAGE_DO_SPACES_REGION - - _APP_STORAGE_DO_SPACES_BUCKET - - _APP_STORAGE_BACKBLAZE_ACCESS_KEY - - _APP_STORAGE_BACKBLAZE_SECRET - - _APP_STORAGE_BACKBLAZE_REGION - - _APP_STORAGE_BACKBLAZE_BUCKET - - _APP_STORAGE_LINODE_ACCESS_KEY - - _APP_STORAGE_LINODE_SECRET - - _APP_STORAGE_LINODE_REGION - - _APP_STORAGE_LINODE_BUCKET - - _APP_STORAGE_WASABI_ACCESS_KEY - - _APP_STORAGE_WASABI_SECRET - - _APP_STORAGE_WASABI_REGION - - _APP_STORAGE_WASABI_BUCKET + - _APP_CONNECTIONS_STORAGE - _APP_FUNCTIONS_SIZE_LIMIT - _APP_FUNCTIONS_TIMEOUT - _APP_FUNCTIONS_BUILD_TIMEOUT @@ -338,27 +318,7 @@ services: - _APP_CONNECTIONS_DB_PROJECT - _APP_CONNECTIONS_CACHE - _APP_CONNECTIONS_QUEUE - - _APP_STORAGE_DEVICE - - _APP_STORAGE_S3_ACCESS_KEY - - _APP_STORAGE_S3_SECRET - - _APP_STORAGE_S3_REGION - - _APP_STORAGE_S3_BUCKET - - _APP_STORAGE_DO_SPACES_ACCESS_KEY - - _APP_STORAGE_DO_SPACES_SECRET - - _APP_STORAGE_DO_SPACES_REGION - - _APP_STORAGE_DO_SPACES_BUCKET - - _APP_STORAGE_BACKBLAZE_ACCESS_KEY - - _APP_STORAGE_BACKBLAZE_SECRET - - _APP_STORAGE_BACKBLAZE_REGION - - _APP_STORAGE_BACKBLAZE_BUCKET - - _APP_STORAGE_LINODE_ACCESS_KEY - - _APP_STORAGE_LINODE_SECRET - - _APP_STORAGE_LINODE_REGION - - _APP_STORAGE_LINODE_BUCKET - - _APP_STORAGE_WASABI_ACCESS_KEY - - _APP_STORAGE_WASABI_SECRET - - _APP_STORAGE_WASABI_REGION - - _APP_STORAGE_WASABI_BUCKET + - _APP_CONNECTIONS_STORAGE - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG - _APP_EXECUTOR_SECRET @@ -740,7 +700,7 @@ services: - openruntimes-functions:/storage/functions:rw - /tmp:/tmp:rw environment: - - OPR_EXECUTOR_CONNECTION_STORAGE=$_APP_FUNCTIONS_CONNECTION_STORAGE + - OPR_EXECUTOR_CONNECTION_STORAGE=$_APP_CONNECTIONS_STORAGE - OPR_EXECUTOR_INACTIVE_TRESHOLD=$_APP_FUNCTIONS_INACTIVE_THRESHOLD - OPR_EXECUTOR_NETWORK=$_APP_FUNCTIONS_RUNTIMES_NETWORK - OPR_EXECUTOR_DOCKER_HUB_USERNAME=$_APP_DOCKER_HUB_USERNAME diff --git a/src/Appwrite/DSN/DSN.php b/src/Appwrite/DSN/DSN.php deleted file mode 100644 index 5605640989..0000000000 --- a/src/Appwrite/DSN/DSN.php +++ /dev/null @@ -1,135 +0,0 @@ -scheme = $parts['scheme'] ?? null; - $this->user = $parts['user'] ?? null; - $this->password = $parts['pass'] ?? null; - $this->host = $parts['host'] ?? null; - $this->port = $parts['port'] ?? null; - $this->database = $parts['path'] ?? null; - $this->query = $parts['query'] ?? null; - } - - /** - * Return the scheme. - * - * @return string - */ - public function getScheme(): string - { - return $this->scheme; - } - - /** - * Return the user. - * - * @return ?string - */ - public function getUser(): ?string - { - return $this->user; - } - - /** - * Return the password. - * - * @return ?string - */ - public function getPassword(): ?string - { - return $this->password; - } - - /** - * Return the host - * - * @return string - */ - public function getHost(): string - { - return $this->host; - } - - /** - * Return the port - * - * @return ?string - */ - public function getPort(): ?string - { - return $this->port; - } - - /** - * Return the database - * - * @return ?string - */ - public function getDatabase(): ?string - { - return ltrim($this->database, '/'); - } - - /** - * Return the query string - * - * @return ?string - */ - public function getQuery(): ?string - { - return $this->query; - } -} diff --git a/src/Appwrite/Resque/Worker.php b/src/Appwrite/Resque/Worker.php index 5d05d77576..8503e18c3c 100644 --- a/src/Appwrite/Resque/Worker.php +++ b/src/Appwrite/Resque/Worker.php @@ -18,6 +18,7 @@ use Utopia\Storage\Device\Backblaze; use Utopia\Storage\Device\S3; use Utopia\Database\Document; use Utopia\Database\Validator\Authorization; +use Utopia\DSN\DSN; abstract class Worker { @@ -276,45 +277,41 @@ abstract class Worker */ public function getDevice($root): Device { - switch (App::getEnv('_APP_STORAGE_DEVICE', Storage::DEVICE_LOCAL)) { - case Storage::DEVICE_LOCAL: + $connection = App::getEnv('_APP_CONNECTIONS_STORAGE', ''); + + $acl = 'private'; + $device = STORAGE_DEVICE_LOCAL; + $accessKey = ''; + $accessSecret = ''; + $bucket = ''; + $region = ''; + + try { + $dsn = new DSN($connection); + $device = $dsn->getScheme(); + $accessKey = $dsn->getUser(); + $accessSecret = $dsn->getPassword(); + $bucket = $dsn->getPath(); + $region = $dsn->getParam('region'); + } catch (\Exception $e) { + Console::error($e->getMessage() . 'Invalid DSN. Defaulting to Local storage.'); + $device = STORAGE_DEVICE_LOCAL; + } + + switch ($device) { + case STORAGE_DEVICE_S3: + return new S3($root, $accessKey, $accessSecret, $bucket, $region, $acl); + case STORAGE_DEVICE_DO_SPACES: + return new DOSpaces($root, $accessKey, $accessSecret, $bucket, $region, $acl); + case STORAGE_DEVICE_BACKBLAZE: + return new Backblaze($root, $accessKey, $accessSecret, $bucket, $region, $acl); + case STORAGE_DEVICE_LINODE: + return new Linode($root, $accessKey, $accessSecret, $bucket, $region, $acl); + case STORAGE_DEVICE_WASABI: + return new Wasabi($root, $accessKey, $accessSecret, $bucket, $region, $acl); + case STORAGE_DEVICE_LOCAL: default: return new Local($root); - case Storage::DEVICE_S3: - $s3AccessKey = App::getEnv('_APP_STORAGE_S3_ACCESS_KEY', ''); - $s3SecretKey = App::getEnv('_APP_STORAGE_S3_SECRET', ''); - $s3Region = App::getEnv('_APP_STORAGE_S3_REGION', ''); - $s3Bucket = App::getEnv('_APP_STORAGE_S3_BUCKET', ''); - $s3Acl = 'private'; - return new S3($root, $s3AccessKey, $s3SecretKey, $s3Bucket, $s3Region, $s3Acl); - case Storage::DEVICE_DO_SPACES: - $doSpacesAccessKey = App::getEnv('_APP_STORAGE_DO_SPACES_ACCESS_KEY', ''); - $doSpacesSecretKey = App::getEnv('_APP_STORAGE_DO_SPACES_SECRET', ''); - $doSpacesRegion = App::getEnv('_APP_STORAGE_DO_SPACES_REGION', ''); - $doSpacesBucket = App::getEnv('_APP_STORAGE_DO_SPACES_BUCKET', ''); - $doSpacesAcl = 'private'; - return new DOSpaces($root, $doSpacesAccessKey, $doSpacesSecretKey, $doSpacesBucket, $doSpacesRegion, $doSpacesAcl); - case Storage::DEVICE_BACKBLAZE: - $backblazeAccessKey = App::getEnv('_APP_STORAGE_BACKBLAZE_ACCESS_KEY', ''); - $backblazeSecretKey = App::getEnv('_APP_STORAGE_BACKBLAZE_SECRET', ''); - $backblazeRegion = App::getEnv('_APP_STORAGE_BACKBLAZE_REGION', ''); - $backblazeBucket = App::getEnv('_APP_STORAGE_BACKBLAZE_BUCKET', ''); - $backblazeAcl = 'private'; - return new Backblaze($root, $backblazeAccessKey, $backblazeSecretKey, $backblazeBucket, $backblazeRegion, $backblazeAcl); - case Storage::DEVICE_LINODE: - $linodeAccessKey = App::getEnv('_APP_STORAGE_LINODE_ACCESS_KEY', ''); - $linodeSecretKey = App::getEnv('_APP_STORAGE_LINODE_SECRET', ''); - $linodeRegion = App::getEnv('_APP_STORAGE_LINODE_REGION', ''); - $linodeBucket = App::getEnv('_APP_STORAGE_LINODE_BUCKET', ''); - $linodeAcl = 'private'; - return new Linode($root, $linodeAccessKey, $linodeSecretKey, $linodeBucket, $linodeRegion, $linodeAcl); - case Storage::DEVICE_WASABI: - $wasabiAccessKey = App::getEnv('_APP_STORAGE_WASABI_ACCESS_KEY', ''); - $wasabiSecretKey = App::getEnv('_APP_STORAGE_WASABI_SECRET', ''); - $wasabiRegion = App::getEnv('_APP_STORAGE_WASABI_REGION', ''); - $wasabiBucket = App::getEnv('_APP_STORAGE_WASABI_BUCKET', ''); - $wasabiAcl = 'private'; - return new Wasabi($root, $wasabiAccessKey, $wasabiSecretKey, $wasabiBucket, $wasabiRegion, $wasabiAcl); } } } diff --git a/tests/unit/DSN/DSNTest.php b/tests/unit/DSN/DSNTest.php deleted file mode 100644 index d1f5ba1197..0000000000 --- a/tests/unit/DSN/DSNTest.php +++ /dev/null @@ -1,81 +0,0 @@ -assertEquals("mariadb", $dsn->getScheme()); - $this->assertEquals("user", $dsn->getUser()); - $this->assertEquals("password", $dsn->getPassword()); - $this->assertEquals("localhost", $dsn->getHost()); - $this->assertEquals("3306", $dsn->getPort()); - $this->assertEquals("database", $dsn->getDatabase()); - $this->assertEquals("charset=utf8&timezone=UTC", $dsn->getQuery()); - - $dsn = new DSN("mariadb://user@localhost:3306/database?charset=utf8&timezone=UTC"); - $this->assertEquals("mariadb", $dsn->getScheme()); - $this->assertEquals("user", $dsn->getUser()); - $this->assertNull($dsn->getPassword()); - $this->assertEquals("localhost", $dsn->getHost()); - $this->assertEquals("3306", $dsn->getPort()); - $this->assertEquals("database", $dsn->getDatabase()); - $this->assertEquals("charset=utf8&timezone=UTC", $dsn->getQuery()); - - $dsn = new DSN("mariadb://user@localhost/database?charset=utf8&timezone=UTC"); - $this->assertEquals("mariadb", $dsn->getScheme()); - $this->assertEquals("user", $dsn->getUser()); - $this->assertNull($dsn->getPassword()); - $this->assertEquals("localhost", $dsn->getHost()); - $this->assertNull($dsn->getPort()); - $this->assertEquals("database", $dsn->getDatabase()); - $this->assertEquals("charset=utf8&timezone=UTC", $dsn->getQuery()); - - $dsn = new DSN("mariadb://user@localhost?charset=utf8&timezone=UTC"); - $this->assertEquals("mariadb", $dsn->getScheme()); - $this->assertEquals("user", $dsn->getUser()); - $this->assertNull($dsn->getPassword()); - $this->assertEquals("localhost", $dsn->getHost()); - $this->assertNull($dsn->getPort()); - $this->assertEmpty($dsn->getDatabase()); - $this->assertEquals("charset=utf8&timezone=UTC", $dsn->getQuery()); - - $dsn = new DSN("mariadb://user@localhost"); - $this->assertEquals("mariadb", $dsn->getScheme()); - $this->assertEquals("user", $dsn->getUser()); - $this->assertNull($dsn->getPassword()); - $this->assertEquals("localhost", $dsn->getHost()); - $this->assertNull($dsn->getPort()); - $this->assertEmpty($dsn->getDatabase()); - $this->assertNull($dsn->getQuery()); - - $dsn = new DSN("mariadb://user:@localhost"); - $this->assertEquals("mariadb", $dsn->getScheme()); - $this->assertEquals("user", $dsn->getUser()); - $this->assertEmpty($dsn->getPassword()); - $this->assertEquals("localhost", $dsn->getHost()); - $this->assertNull($dsn->getPort()); - $this->assertEmpty($dsn->getDatabase()); - $this->assertNull($dsn->getQuery()); - - $dsn = new DSN("mariadb://localhost"); - $this->assertEquals("mariadb", $dsn->getScheme()); - $this->assertNull($dsn->getUser()); - $this->assertNull($dsn->getPassword()); - $this->assertEquals("localhost", $dsn->getHost()); - $this->assertNull($dsn->getPort()); - $this->assertEmpty($dsn->getDatabase()); - $this->assertNull($dsn->getQuery()); - } - - public function testFail(): void - { - $this->expectException(\InvalidArgumentException::class); - new DSN("mariadb://"); - } -}