CodeMirror/appwrite
1
0
Fork 0
mirror of synced 2024-07-02 05:00:33 +12:00
Code Issues Projects Releases Wiki Activity

Merge pull request #4578 from appwrite/feat-support-dsn

Browse source 
feat: add support for the new DSN class
This commit is contained in:
Christy Jacob 2022-11-17 13:34:30 +05:30 committed by GitHub
parent 2a9345cbcd b989a35b62
commit 6acb07109a
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 132 additions and 452 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

23
.env
View file

@ -29,27 +29,7 @@ _APP_CONNECTIONS_DB_CONSOLE=db_fra1_01=mariadb://user:password@mariadb:3306/appw
_APP_CONNECTIONS_CACHE=redis_fra1_01=redis://redis:6379
_APP_CONNECTIONS_QUEUE=redis_fra1_01=redis://redis:6379
_APP_CONNECTIONS_PUBSUB=redis_fra1_01=redis://redis:6379
_APP_STORAGE_DEVICE=Local
_APP_STORAGE_S3_ACCESS_KEY=
_APP_STORAGE_S3_SECRET=
_APP_STORAGE_S3_REGION=us-east-1
_APP_STORAGE_S3_BUCKET=
_APP_STORAGE_DO_SPACES_ACCESS_KEY=
_APP_STORAGE_DO_SPACES_SECRET=
_APP_STORAGE_DO_SPACES_REGION=us-east-1
_APP_STORAGE_DO_SPACES_BUCKET=
_APP_STORAGE_BACKBLAZE_ACCESS_KEY=
_APP_STORAGE_BACKBLAZE_SECRET=
_APP_STORAGE_BACKBLAZE_REGION=us-west-004
_APP_STORAGE_BACKBLAZE_BUCKET=
_APP_STORAGE_LINODE_ACCESS_KEY=
_APP_STORAGE_LINODE_SECRET=
_APP_STORAGE_LINODE_REGION=eu-central-1
_APP_STORAGE_LINODE_BUCKET=
_APP_STORAGE_WASABI_ACCESS_KEY=
_APP_STORAGE_WASABI_SECRET=
_APP_STORAGE_WASABI_REGION=eu-central-1
_APP_STORAGE_WASABI_BUCKET=
_APP_CONNECTIONS_STORAGE=file://localhost
_APP_STORAGE_ANTIVIRUS=disabled
_APP_STORAGE_ANTIVIRUS_HOST=clamav
_APP_STORAGE_ANTIVIRUS_PORT=3310
@ -73,7 +53,6 @@ _APP_FUNCTIONS_CPUS=1
_APP_FUNCTIONS_MEMORY=512
_APP_FUNCTIONS_INACTIVE_THRESHOLD=600
_APP_FUNCTIONS_RUNTIMES_NETWORK=openruntimes-runtimes
_APP_FUNCTIONS_CONNECTION_STORAGE=file://localhost
_APP_EXECUTOR_SECRET=your-secret-key
_APP_EXECUTOR_HOST=http://exc1/v1
_APP_FUNCTIONS_RUNTIMES=

22
Dockerfile
View file

@ -205,27 +205,7 @@ ENV _APP_SERVER=swoole \
_APP_STORAGE_ANTIVIRUS=enabled \
_APP_STORAGE_ANTIVIRUS_HOST=clamav \
_APP_STORAGE_ANTIVIRUS_PORT=3310 \
_APP_STORAGE_DEVICE=Local \
_APP_STORAGE_S3_ACCESS_KEY= \
_APP_STORAGE_S3_SECRET= \
_APP_STORAGE_S3_REGION= \
_APP_STORAGE_S3_BUCKET= \
_APP_STORAGE_DO_SPACES_ACCESS_KEY= \
_APP_STORAGE_DO_SPACES_SECRET= \
_APP_STORAGE_DO_SPACES_REGION= \
_APP_STORAGE_DO_SPACES_BUCKET= \
_APP_STORAGE_BACKBLAZE_ACCESS_KEY= \
_APP_STORAGE_BACKBLAZE_SECRET= \
_APP_STORAGE_BACKBLAZE_REGION= \
_APP_STORAGE_BACKBLAZE_BUCKET= \
_APP_STORAGE_LINODE_ACCESS_KEY= \
_APP_STORAGE_LINODE_SECRET= \
_APP_STORAGE_LINODE_REGION= \
_APP_STORAGE_LINODE_BUCKET= \
_APP_STORAGE_WASABI_ACCESS_KEY= \
_APP_STORAGE_WASABI_SECRET= \
_APP_STORAGE_WASABI_REGION= \
_APP_STORAGE_WASABI_BUCKET= \
_APP_CONNECTIONS_STORAGE= \
_APP_REDIS_HOST=redis \
_APP_REDIS_PORT=6379 \
_APP_DB_HOST=mariadb \

59
app/config/variables.php
View file

@ -507,9 +507,17 @@ return [
'question' => '',
'filter' => ''
],
[
'name' => '_APP_CONNECTIONS_STORAGE',
'description' => 'A DSN representing the storage device to connect to. The DSN takes the following format <device>://<access_key>:<access_secret>@<host>:<port>/<bucket>?region=<region>. For example, for S3: \'s3://access_key:access_secret@host:port/bucket?region=us-east-1\'. To use the local filesystem, you can leave this variable empty. Available devices are file, s3, dospaces, linode, backblaze and wasabi.',
'introduction' => '1.1.0',
'default' => '',
'required' => false,
'question' => '',
],
[
'name' => '_APP_STORAGE_DEVICE',
'description' => 'Select default storage device. The default value is \'Local\'. List of supported adapters are \'Local\', \'S3\', \'DOSpaces\', \'Backblaze\', \'Linode\' and \'Wasabi\'.',
'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.',
'introduction' => '0.13.0',
'default' => 'Local',
'required' => false,
@ -517,7 +525,7 @@ return [
],
[
'name' => '_APP_STORAGE_S3_ACCESS_KEY',
'description' => 'AWS S3 storage access key. Required when the storage adapter is set to S3. You can get your access key from your AWS console',
'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.',
'introduction' => '0.13.0',
'default' => '',
'required' => false,
@ -525,7 +533,7 @@ return [
],
[
'name' => '_APP_STORAGE_S3_SECRET',
'description' => 'AWS S3 storage secret key. Required when the storage adapter is set to S3. You can get your secret key from your AWS console.',
'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.',
'introduction' => '0.13.0',
'default' => '',
'required' => false,
@ -533,7 +541,7 @@ return [
],
[
'name' => '_APP_STORAGE_S3_REGION',
'description' => 'AWS S3 storage region. Required when storage adapter is set to S3. You can find your region info for your bucket from AWS console.',
'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.',
'introduction' => '0.13.0',
'default' => 'us-east-1',
'required' => false,
@ -541,7 +549,7 @@ return [
],
[
'name' => '_APP_STORAGE_S3_BUCKET',
'description' => 'AWS S3 storage bucket. Required when storage adapter is set to S3. You can create buckets in your AWS console.',
'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.',
'introduction' => '0.13.0',
'default' => '',
'required' => false,
@ -549,7 +557,7 @@ return [
],
[
'name' => '_APP_STORAGE_DO_SPACES_ACCESS_KEY',
'description' => 'DigitalOcean spaces access key. Required when the storage adapter is set to DOSpaces. You can get your access key from your DigitalOcean console.',
'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.',
'introduction' => '0.13.0',
'default' => '',
'required' => false,
@ -557,7 +565,7 @@ return [
],
[
'name' => '_APP_STORAGE_DO_SPACES_SECRET',
'description' => 'DigitalOcean spaces secret key. Required when the storage adapter is set to DOSpaces. You can get your secret key from your DigitalOcean console.',
'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.',
'introduction' => '0.13.0',
'default' => '',
'required' => false,
@ -565,7 +573,7 @@ return [
],
[
'name' => '_APP_STORAGE_DO_SPACES_REGION',
'description' => 'DigitalOcean spaces region. Required when storage adapter is set to DOSpaces. You can find your region info for your space from DigitalOcean console.',
'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.',
'introduction' => '0.13.0',
'default' => 'us-east-1',
'required' => false,
@ -573,7 +581,7 @@ return [
],
[
'name' => '_APP_STORAGE_DO_SPACES_BUCKET',
'description' => 'DigitalOcean spaces bucket. Required when storage adapter is set to DOSpaces. You can create spaces in your DigitalOcean console.',
'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.',
'introduction' => '0.13.0',
'default' => '',
'required' => false,
@ -581,7 +589,7 @@ return [
],
[
'name' => '_APP_STORAGE_BACKBLAZE_ACCESS_KEY',
'description' => 'Backblaze access key. Required when the storage adapter is set to Backblaze. Your Backblaze keyID will be your access key. You can get your keyID from your Backblaze console.',
'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.',
'introduction' => '0.14.2',
'default' => '',
'required' => false,
@ -589,7 +597,7 @@ return [
],
[
'name' => '_APP_STORAGE_BACKBLAZE_SECRET',
'description' => 'Backblaze secret key. Required when the storage adapter is set to Backblaze. Your Backblaze applicationKey will be your secret key. You can get your applicationKey from your Backblaze console.',
'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.',
'introduction' => '0.14.2',
'default' => '',
'required' => false,
@ -597,7 +605,7 @@ return [
],
[
'name' => '_APP_STORAGE_BACKBLAZE_REGION',
'description' => 'Backblaze region. Required when storage adapter is set to Backblaze. You can find your region info from your Backblaze console.',
'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.',
'introduction' => '0.14.2',
'default' => 'us-west-004',
'required' => false,
@ -605,7 +613,7 @@ return [
],
[
'name' => '_APP_STORAGE_BACKBLAZE_BUCKET',
'description' => 'Backblaze bucket. Required when storage adapter is set to Backblaze. You can create your bucket from your Backblaze console.',
'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.',
'introduction' => '0.14.2',
'default' => '',
'required' => false,
@ -613,7 +621,7 @@ return [
],
[
'name' => '_APP_STORAGE_LINODE_ACCESS_KEY',
'description' => 'Linode object storage access key. Required when the storage adapter is set to Linode. You can get your access key from your Linode console.',
'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.',
'introduction' => '0.14.2',
'default' => '',
'required' => false,
@ -621,7 +629,7 @@ return [
],
[
'name' => '_APP_STORAGE_LINODE_SECRET',
'description' => 'Linode object storage secret key. Required when the storage adapter is set to Linode. You can get your secret key from your Linode console.',
'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.',
'introduction' => '0.14.2',
'default' => '',
'required' => false,
@ -629,7 +637,7 @@ return [
],
[
'name' => '_APP_STORAGE_LINODE_REGION',
'description' => 'Linode object storage region. Required when storage adapter is set to Linode. You can find your region info from your Linode console.',
'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.',
'introduction' => '0.14.2',
'default' => 'eu-central-1',
'required' => false,
@ -637,7 +645,7 @@ return [
],
[
'name' => '_APP_STORAGE_LINODE_BUCKET',
'description' => 'Linode object storage bucket. Required when storage adapter is set to Linode. You can create buckets in your Linode console.',
'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.',
'introduction' => '0.14.2',
'default' => '',
'required' => false,
@ -645,7 +653,7 @@ return [
],
[
'name' => '_APP_STORAGE_WASABI_ACCESS_KEY',
'description' => 'Wasabi access key. Required when the storage adapter is set to Wasabi. You can get your access key from your Wasabi console.',
'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.',
'introduction' => '0.14.2',
'default' => '',
'required' => false,
@ -653,7 +661,7 @@ return [
],
[
'name' => '_APP_STORAGE_WASABI_SECRET',
'description' => 'Wasabi secret key. Required when the storage adapter is set to Wasabi. You can get your secret key from your Wasabi console.',
'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.',
'introduction' => '0.14.2',
'default' => '',
'required' => false,
@ -661,7 +669,7 @@ return [
],
[
'name' => '_APP_STORAGE_WASABI_REGION',
'description' => 'Wasabi region. Required when storage adapter is set to Wasabi. You can find your region info from your Wasabi console.',
'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.',
'introduction' => '0.14.2',
'default' => 'eu-central-1',
'required' => false,
@ -669,7 +677,7 @@ return [
],
[
'name' => '_APP_STORAGE_WASABI_BUCKET',
'description' => 'Wasabi bucket. Required when storage adapter is set to Wasabi. You can create buckets in your Wasabi console.',
'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.',
'introduction' => '0.14.2',
'default' => '',
'required' => false,
@ -860,15 +868,6 @@ return [
'required' => false,
'question' => '',
'filter' => ''
],
[
'name' => '_APP_FUNCTIONS_CONNECTION_STORAGE',
'description' => 'DSN record of storage driver where Open Runtimes Executor stores output of Appwrite Function Deployment builds.',
'introduction' => '1.2.0',
'default' => '',
'required' => false,
'question' => '',
'filter' => ''
]
],
],

11
app/controllers/api/storage.php
View file

@ -494,7 +494,16 @@ App::post('/v1/storage/buckets/:bucketId/files')
}
if ($chunksUploaded === $chunks) {
if (App::getEnv('_APP_STORAGE_ANTIVIRUS') === 'enabled' && $bucket->getAttribute('antivirus', true) && $fileSize <= APP_LIMIT_ANTIVIRUS && App::getEnv('_APP_STORAGE_DEVICE', Storage::DEVICE_LOCAL) === Storage::DEVICE_LOCAL) {
$connection = App::getEnv('_APP_CONNECTIONS_STORAGE', ''); /** @TODO : move this to the registry or someplace else */
$device = STORAGE_DEVICE_LOCAL;
try {
$dsn = new DSN($connection);
$device = $dsn->getScheme();
} catch (\Exception $e) {
$device = STORAGE_DEVICE_LOCAL;
}
if (App::getEnv('_APP_STORAGE_ANTIVIRUS') === 'enabled' && $bucket->getAttribute('antivirus', true) && $fileSize <= APP_LIMIT_ANTIVIRUS && $device === STORAGE_DEVICE_LOCAL) {
$antivirus = new Network(
App::getEnv('_APP_STORAGE_ANTIVIRUS_HOST', 'clamav'),
(int) App::getEnv('_APP_STORAGE_ANTIVIRUS_PORT', 3310)

79
app/init.php
View file

@ -26,7 +26,6 @@ use Appwrite\SMS\Adapter\TextMagic;
use Appwrite\SMS\Adapter\Twilio;
use Appwrite\SMS\Adapter\Msg91;
use Appwrite\SMS\Adapter\Vonage;
use Appwrite\DSN\DSN;
use Appwrite\Event\Audit;
use Appwrite\Event\Database as EventDatabase;
use Appwrite\Event\Event;
@ -56,6 +55,7 @@ use Utopia\Locale\Locale;
use Utopia\Registry\Registry;
use Utopia\Storage\Device;
use Utopia\Storage\Storage;
use Utopia\DSN\DSN;
use Utopia\Storage\Device\Backblaze;
use Utopia\Storage\Device\DOSpaces;
use Utopia\Storage\Device\Local;
@ -160,6 +160,13 @@ const DELETE_TYPE_SCHEDULES = 'schedules';
const COMPRESSION_TYPE_NONE = 'none';
const COMPRESSION_TYPE_GZIP = 'gzip';
const COMPRESSION_TYPE_ZSTD = 'zstd';
// Storage Device Types
const STORAGE_DEVICE_LOCAL = 'file';
const STORAGE_DEVICE_S3 = 's3';
const STORAGE_DEVICE_DO_SPACES = 'dospaces';
const STORAGE_DEVICE_BACKBLAZE = 'backblaze';
const STORAGE_DEVICE_LINODE = 'linode';
const STORAGE_DEVICE_WASABI = 'wasabi';
// Mail Types
const MAIL_TYPE_VERIFICATION = 'verification';
const MAIL_TYPE_MAGIC_SESSION = 'magicSession';
@ -1099,45 +1106,41 @@ App::setResource('deviceBuilds', function ($project) {
function getDevice($root): Device
{
switch (App::getEnv('_APP_STORAGE_DEVICE', Storage::DEVICE_LOCAL)) {
case Storage::DEVICE_LOCAL:
$connection = App::getEnv('_APP_CONNECTIONS_STORAGE', '');
$acl = 'private';
$device = STORAGE_DEVICE_LOCAL;
$accessKey = '';
$accessSecret = '';
$bucket = '';
$region = '';
try {
$dsn = new DSN($connection);
$device = $dsn->getScheme();
$accessKey = $dsn->getUser();
$accessSecret = $dsn->getPassword();
$bucket = $dsn->getPath();
$region = $dsn->getParam('region');
} catch (\Exception $e) {
Console::eor($e->getMessage() . 'Invalid DSN. Defaulting to Local storage.');
$device = STORAGE_DEVICE_LOCAL;
}
switch ($device) {
case STORAGE_DEVICE_S3:
return new S3($root, $accessKey, $accessSecret, $bucket, $region, $acl);
case STORAGE_DEVICE_DO_SPACES:
return new DOSpaces($root, $accessKey, $accessSecret, $bucket, $region, $acl);
case STORAGE_DEVICE_BACKBLAZE:
return new Backblaze($root, $accessKey, $accessSecret, $bucket, $region, $acl);
case STORAGE_DEVICE_LINODE:
return new Linode($root, $accessKey, $accessSecret, $bucket, $region, $acl);
case STORAGE_DEVICE_WASABI:
return new Wasabi($root, $accessKey, $accessSecret, $bucket, $region, $acl);
case STORAGE_DEVICE_LOCAL:
default:
return new Local($root);
case Storage::DEVICE_S3:
$s3AccessKey = App::getEnv('_APP_STORAGE_S3_ACCESS_KEY', '');
$s3SecretKey = App::getEnv('_APP_STORAGE_S3_SECRET', '');
$s3Region = App::getEnv('_APP_STORAGE_S3_REGION', '');
$s3Bucket = App::getEnv('_APP_STORAGE_S3_BUCKET', '');
$s3Acl = 'private';
return new S3($root, $s3AccessKey, $s3SecretKey, $s3Bucket, $s3Region, $s3Acl);
case Storage::DEVICE_DO_SPACES:
$doSpacesAccessKey = App::getEnv('_APP_STORAGE_DO_SPACES_ACCESS_KEY', '');
$doSpacesSecretKey = App::getEnv('_APP_STORAGE_DO_SPACES_SECRET', '');
$doSpacesRegion = App::getEnv('_APP_STORAGE_DO_SPACES_REGION', '');
$doSpacesBucket = App::getEnv('_APP_STORAGE_DO_SPACES_BUCKET', '');
$doSpacesAcl = 'private';
return new DOSpaces($root, $doSpacesAccessKey, $doSpacesSecretKey, $doSpacesBucket, $doSpacesRegion, $doSpacesAcl);
case Storage::DEVICE_BACKBLAZE:
$backblazeAccessKey = App::getEnv('_APP_STORAGE_BACKBLAZE_ACCESS_KEY', '');
$backblazeSecretKey = App::getEnv('_APP_STORAGE_BACKBLAZE_SECRET', '');
$backblazeRegion = App::getEnv('_APP_STORAGE_BACKBLAZE_REGION', '');
$backblazeBucket = App::getEnv('_APP_STORAGE_BACKBLAZE_BUCKET', '');
$backblazeAcl = 'private';
return new Backblaze($root, $backblazeAccessKey, $backblazeSecretKey, $backblazeBucket, $backblazeRegion, $backblazeAcl);
case Storage::DEVICE_LINODE:
$linodeAccessKey = App::getEnv('_APP_STORAGE_LINODE_ACCESS_KEY', '');
$linodeSecretKey = App::getEnv('_APP_STORAGE_LINODE_SECRET', '');
$linodeRegion = App::getEnv('_APP_STORAGE_LINODE_REGION', '');
$linodeBucket = App::getEnv('_APP_STORAGE_LINODE_BUCKET', '');
$linodeAcl = 'private';
return new Linode($root, $linodeAccessKey, $linodeSecretKey, $linodeBucket, $linodeRegion, $linodeAcl);
case Storage::DEVICE_WASABI:
$wasabiAccessKey = App::getEnv('_APP_STORAGE_WASABI_ACCESS_KEY', '');
$wasabiSecretKey = App::getEnv('_APP_STORAGE_WASABI_SECRET', '');
$wasabiRegion = App::getEnv('_APP_STORAGE_WASABI_REGION', '');
$wasabiBucket = App::getEnv('_APP_STORAGE_WASABI_BUCKET', '');
$wasabiAcl = 'private';
return new Wasabi($root, $wasabiAccessKey, $wasabiSecretKey, $wasabiBucket, $wasabiRegion, $wasabiAcl);
}
}

46
app/views/install/compose.phtml
View file

@ -120,27 +120,7 @@ services:
- _APP_STORAGE_ANTIVIRUS
- _APP_STORAGE_ANTIVIRUS_HOST
- _APP_STORAGE_ANTIVIRUS_PORT
- _APP_STORAGE_DEVICE
- _APP_STORAGE_S3_ACCESS_KEY
- _APP_STORAGE_S3_SECRET
- _APP_STORAGE_S3_REGION
- _APP_STORAGE_S3_BUCKET
- _APP_STORAGE_DO_SPACES_ACCESS_KEY
- _APP_STORAGE_DO_SPACES_SECRET
- _APP_STORAGE_DO_SPACES_REGION
- _APP_STORAGE_DO_SPACES_BUCKET
- _APP_STORAGE_BACKBLAZE_ACCESS_KEY
- _APP_STORAGE_BACKBLAZE_SECRET
- _APP_STORAGE_BACKBLAZE_REGION
- _APP_STORAGE_BACKBLAZE_BUCKET
- _APP_STORAGE_LINODE_ACCESS_KEY
- _APP_STORAGE_LINODE_SECRET
- _APP_STORAGE_LINODE_REGION
- _APP_STORAGE_LINODE_BUCKET
- _APP_STORAGE_WASABI_ACCESS_KEY
- _APP_STORAGE_WASABI_SECRET
- _APP_STORAGE_WASABI_REGION
- _APP_STORAGE_WASABI_BUCKET
- _APP_CONNECTIONS_STORAGE
- _APP_FUNCTIONS_SIZE_LIMIT
- _APP_FUNCTIONS_TIMEOUT
- _APP_FUNCTIONS_BUILD_TIMEOUT
@ -284,27 +264,7 @@ services:
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_STORAGE_DEVICE
- _APP_STORAGE_S3_ACCESS_KEY
- _APP_STORAGE_S3_SECRET
- _APP_STORAGE_S3_REGION
- _APP_STORAGE_S3_BUCKET
- _APP_STORAGE_DO_SPACES_ACCESS_KEY
- _APP_STORAGE_DO_SPACES_SECRET
- _APP_STORAGE_DO_SPACES_REGION
- _APP_STORAGE_DO_SPACES_BUCKET
- _APP_STORAGE_BACKBLAZE_ACCESS_KEY
- _APP_STORAGE_BACKBLAZE_SECRET
- _APP_STORAGE_BACKBLAZE_REGION
- _APP_STORAGE_BACKBLAZE_BUCKET
- _APP_STORAGE_LINODE_ACCESS_KEY
- _APP_STORAGE_LINODE_SECRET
- _APP_STORAGE_LINODE_REGION
- _APP_STORAGE_LINODE_BUCKET
- _APP_STORAGE_WASABI_ACCESS_KEY
- _APP_STORAGE_WASABI_SECRET
- _APP_STORAGE_WASABI_REGION
- _APP_STORAGE_WASABI_BUCKET
- _APP_CONNECTIONS_STORAGE
- _APP_LOGGING_PROVIDER
- _APP_LOGGING_CONFIG
- _APP_EXECUTOR_SECRET
@ -598,7 +558,7 @@ services:
- openruntimes-functions:/storage/functions:rw
- /tmp:/tmp:rw
environment:
- OPR_EXECUTOR_CONNECTION_STORAGE=$_APP_FUNCTIONS_CONNECTION_STORAGE
- OPR_EXECUTOR_CONNECTION_STORAGE=$_APP_CONNECTIONS_STORAGE
- OPR_EXECUTOR_INACTIVE_TRESHOLD=$_APP_FUNCTIONS_INACTIVE_THRESHOLD
- OPR_EXECUTOR_NETWORK=$_APP_FUNCTIONS_RUNTIMES_NETWORK
- OPR_EXECUTOR_DOCKER_HUB_USERNAME=$_APP_DOCKER_HUB_USERNAME

11
app/workers/builds.php
View file

@ -79,6 +79,15 @@ class BuildsV1 extends Worker
throw new Exception('Runtime "' . $function->getAttribute('runtime', '') . '" is not supported');
}
$connection = App::getEnv('_APP_CONNECTIONS_STORAGE', ''); /** @TODO : move this to the registry or someplace else */
$device = STORAGE_DEVICE_LOCAL;
try {
$dsn = new DSN($connection);
$device = $dsn->getScheme();
} catch (\Exception $e) {
$device = STORAGE_DEVICE_LOCAL;
}
$buildId = $deployment->getAttribute('buildId', '');
$startTime = DateTime::now();
if (empty($buildId)) {
@ -92,7 +101,7 @@ class BuildsV1 extends Worker
'outputPath' => '',
'runtime' => $function->getAttribute('runtime'),
'source' => $deployment->getAttribute('path'),
'sourceType' => App::getEnv('_APP_STORAGE_DEVICE', Storage::DEVICE_LOCAL),
'sourceType' => $device,
'stdout' => '',
'stderr' => '',
'endTime' => null,

46
docker-compose.yml
View file

@ -146,27 +146,7 @@ services:
- _APP_STORAGE_ANTIVIRUS
- _APP_STORAGE_ANTIVIRUS_HOST
- _APP_STORAGE_ANTIVIRUS_PORT
- _APP_STORAGE_DEVICE
- _APP_STORAGE_S3_ACCESS_KEY
- _APP_STORAGE_S3_SECRET
- _APP_STORAGE_S3_REGION
- _APP_STORAGE_S3_BUCKET
- _APP_STORAGE_DO_SPACES_ACCESS_KEY
- _APP_STORAGE_DO_SPACES_SECRET
- _APP_STORAGE_DO_SPACES_REGION
- _APP_STORAGE_DO_SPACES_BUCKET
- _APP_STORAGE_BACKBLAZE_ACCESS_KEY
- _APP_STORAGE_BACKBLAZE_SECRET
- _APP_STORAGE_BACKBLAZE_REGION
- _APP_STORAGE_BACKBLAZE_BUCKET
- _APP_STORAGE_LINODE_ACCESS_KEY
- _APP_STORAGE_LINODE_SECRET
- _APP_STORAGE_LINODE_REGION
- _APP_STORAGE_LINODE_BUCKET
- _APP_STORAGE_WASABI_ACCESS_KEY
- _APP_STORAGE_WASABI_SECRET
- _APP_STORAGE_WASABI_REGION
- _APP_STORAGE_WASABI_BUCKET
- _APP_CONNECTIONS_STORAGE
- _APP_FUNCTIONS_SIZE_LIMIT
- _APP_FUNCTIONS_TIMEOUT
- _APP_FUNCTIONS_BUILD_TIMEOUT
@ -338,27 +318,7 @@ services:
- _APP_CONNECTIONS_DB_PROJECT
- _APP_CONNECTIONS_CACHE
- _APP_CONNECTIONS_QUEUE
- _APP_STORAGE_DEVICE
- _APP_STORAGE_S3_ACCESS_KEY
- _APP_STORAGE_S3_SECRET
- _APP_STORAGE_S3_REGION
- _APP_STORAGE_S3_BUCKET
- _APP_STORAGE_DO_SPACES_ACCESS_KEY
- _APP_STORAGE_DO_SPACES_SECRET
- _APP_STORAGE_DO_SPACES_REGION
- _APP_STORAGE_DO_SPACES_BUCKET
- _APP_STORAGE_BACKBLAZE_ACCESS_KEY
- _APP_STORAGE_BACKBLAZE_SECRET
- _APP_STORAGE_BACKBLAZE_REGION
- _APP_STORAGE_BACKBLAZE_BUCKET
- _APP_STORAGE_LINODE_ACCESS_KEY
- _APP_STORAGE_LINODE_SECRET
- _APP_STORAGE_LINODE_REGION
- _APP_STORAGE_LINODE_BUCKET
- _APP_STORAGE_WASABI_ACCESS_KEY
- _APP_STORAGE_WASABI_SECRET
- _APP_STORAGE_WASABI_REGION
- _APP_STORAGE_WASABI_BUCKET
- _APP_CONNECTIONS_STORAGE
- _APP_LOGGING_PROVIDER
- _APP_LOGGING_CONFIG
- _APP_EXECUTOR_SECRET
@ -740,7 +700,7 @@ services:
- openruntimes-functions:/storage/functions:rw
- /tmp:/tmp:rw
environment:
- OPR_EXECUTOR_CONNECTION_STORAGE=$_APP_FUNCTIONS_CONNECTION_STORAGE
- OPR_EXECUTOR_CONNECTION_STORAGE=$_APP_CONNECTIONS_STORAGE
- OPR_EXECUTOR_INACTIVE_TRESHOLD=$_APP_FUNCTIONS_INACTIVE_THRESHOLD
- OPR_EXECUTOR_NETWORK=$_APP_FUNCTIONS_RUNTIMES_NETWORK
- OPR_EXECUTOR_DOCKER_HUB_USERNAME=$_APP_DOCKER_HUB_USERNAME

135
src/Appwrite/DSN/DSN.php
View file

@ -1,135 +0,0 @@
<?php
namespace Appwrite\DSN;
class DSN
{
/**
* @var string
*/
protected string $scheme;
/**
* @var ?string
*/
protected ?string $user;
/**
* @var ?string
*/
protected ?string $password;
/**
* @var string
*/
protected string $host;
/**
* @var ?string
*/
protected ?string $port;
/**
* @var ?string
*/
protected ?string $database;
/**
* @var ?string
*/
protected ?string $query;
/**
* Construct
*
* Construct a new DSN object
*
* @param string $dsn
*/
public function __construct(string $dsn)
{
$parts = parse_url($dsn);
if (!$parts) {
throw new \InvalidArgumentException("Unable to parse DSN: $dsn");
}
$this->scheme = $parts['scheme'] ?? null;
$this->user = $parts['user'] ?? null;
$this->password = $parts['pass'] ?? null;
$this->host = $parts['host'] ?? null;
$this->port = $parts['port'] ?? null;
$this->database = $parts['path'] ?? null;
$this->query = $parts['query'] ?? null;
}
/**
* Return the scheme.
*
* @return string
*/
public function getScheme(): string
{
return $this->scheme;
}
/**
* Return the user.
*
* @return ?string
*/
public function getUser(): ?string
{
return $this->user;
}
/**
* Return the password.
*
* @return ?string
*/
public function getPassword(): ?string
{
return $this->password;
}
/**
* Return the host
*
* @return string
*/
public function getHost(): string
{
return $this->host;
}
/**
* Return the port
*
* @return ?string
*/
public function getPort(): ?string
{
return $this->port;
}
/**
* Return the database
*
* @return ?string
*/
public function getDatabase(): ?string
{
return ltrim($this->database, '/');
}
/**
* Return the query string
*
* @return ?string
*/
public function getQuery(): ?string
{
return $this->query;
}
}

71
src/Appwrite/Resque/Worker.php
View file

@ -18,6 +18,7 @@ use Utopia\Storage\Device\Backblaze;
use Utopia\Storage\Device\S3;
use Utopia\Database\Document;
use Utopia\Database\Validator\Authorization;
use Utopia\DSN\DSN;
abstract class Worker
{
@ -276,45 +277,41 @@ abstract class Worker
*/
public function getDevice($root): Device
{
switch (App::getEnv('_APP_STORAGE_DEVICE', Storage::DEVICE_LOCAL)) {
case Storage::DEVICE_LOCAL:
$connection = App::getEnv('_APP_CONNECTIONS_STORAGE', '');
$acl = 'private';
$device = STORAGE_DEVICE_LOCAL;
$accessKey = '';
$accessSecret = '';
$bucket = '';
$region = '';
try {
$dsn = new DSN($connection);
$device = $dsn->getScheme();
$accessKey = $dsn->getUser();
$accessSecret = $dsn->getPassword();
$bucket = $dsn->getPath();
$region = $dsn->getParam('region');
} catch (\Exception $e) {
Console::error($e->getMessage() . 'Invalid DSN. Defaulting to Local storage.');
$device = STORAGE_DEVICE_LOCAL;
}
switch ($device) {
case STORAGE_DEVICE_S3:
return new S3($root, $accessKey, $accessSecret, $bucket, $region, $acl);
case STORAGE_DEVICE_DO_SPACES:
return new DOSpaces($root, $accessKey, $accessSecret, $bucket, $region, $acl);
case STORAGE_DEVICE_BACKBLAZE:
return new Backblaze($root, $accessKey, $accessSecret, $bucket, $region, $acl);
case STORAGE_DEVICE_LINODE:
return new Linode($root, $accessKey, $accessSecret, $bucket, $region, $acl);
case STORAGE_DEVICE_WASABI:
return new Wasabi($root, $accessKey, $accessSecret, $bucket, $region, $acl);
case STORAGE_DEVICE_LOCAL:
default:
return new Local($root);
case Storage::DEVICE_S3:
$s3AccessKey = App::getEnv('_APP_STORAGE_S3_ACCESS_KEY', '');
$s3SecretKey = App::getEnv('_APP_STORAGE_S3_SECRET', '');
$s3Region = App::getEnv('_APP_STORAGE_S3_REGION', '');
$s3Bucket = App::getEnv('_APP_STORAGE_S3_BUCKET', '');
$s3Acl = 'private';
return new S3($root, $s3AccessKey, $s3SecretKey, $s3Bucket, $s3Region, $s3Acl);
case Storage::DEVICE_DO_SPACES:
$doSpacesAccessKey = App::getEnv('_APP_STORAGE_DO_SPACES_ACCESS_KEY', '');
$doSpacesSecretKey = App::getEnv('_APP_STORAGE_DO_SPACES_SECRET', '');
$doSpacesRegion = App::getEnv('_APP_STORAGE_DO_SPACES_REGION', '');
$doSpacesBucket = App::getEnv('_APP_STORAGE_DO_SPACES_BUCKET', '');
$doSpacesAcl = 'private';
return new DOSpaces($root, $doSpacesAccessKey, $doSpacesSecretKey, $doSpacesBucket, $doSpacesRegion, $doSpacesAcl);
case Storage::DEVICE_BACKBLAZE:
$backblazeAccessKey = App::getEnv('_APP_STORAGE_BACKBLAZE_ACCESS_KEY', '');
$backblazeSecretKey = App::getEnv('_APP_STORAGE_BACKBLAZE_SECRET', '');
$backblazeRegion = App::getEnv('_APP_STORAGE_BACKBLAZE_REGION', '');
$backblazeBucket = App::getEnv('_APP_STORAGE_BACKBLAZE_BUCKET', '');
$backblazeAcl = 'private';
return new Backblaze($root, $backblazeAccessKey, $backblazeSecretKey, $backblazeBucket, $backblazeRegion, $backblazeAcl);
case Storage::DEVICE_LINODE:
$linodeAccessKey = App::getEnv('_APP_STORAGE_LINODE_ACCESS_KEY', '');
$linodeSecretKey = App::getEnv('_APP_STORAGE_LINODE_SECRET', '');
$linodeRegion = App::getEnv('_APP_STORAGE_LINODE_REGION', '');
$linodeBucket = App::getEnv('_APP_STORAGE_LINODE_BUCKET', '');
$linodeAcl = 'private';
return new Linode($root, $linodeAccessKey, $linodeSecretKey, $linodeBucket, $linodeRegion, $linodeAcl);
case Storage::DEVICE_WASABI:
$wasabiAccessKey = App::getEnv('_APP_STORAGE_WASABI_ACCESS_KEY', '');
$wasabiSecretKey = App::getEnv('_APP_STORAGE_WASABI_SECRET', '');
$wasabiRegion = App::getEnv('_APP_STORAGE_WASABI_REGION', '');
$wasabiBucket = App::getEnv('_APP_STORAGE_WASABI_BUCKET', '');
$wasabiAcl = 'private';
return new Wasabi($root, $wasabiAccessKey, $wasabiSecretKey, $wasabiBucket, $wasabiRegion, $wasabiAcl);
}
}
}

81
tests/unit/DSN/DSNTest.php
View file

@ -1,81 +0,0 @@
<?php
namespace Tests\Unit\DSN;
use Appwrite\DSN\DSN;
use PHPUnit\Framework\TestCase;
class DSNTest extends TestCase
{
public function testSuccess(): void
{
$dsn = new DSN("mariadb://user:password@localhost:3306/database?charset=utf8&timezone=UTC");
$this->assertEquals("mariadb", $dsn->getScheme());
$this->assertEquals("user", $dsn->getUser());
$this->assertEquals("password", $dsn->getPassword());
$this->assertEquals("localhost", $dsn->getHost());
$this->assertEquals("3306", $dsn->getPort());
$this->assertEquals("database", $dsn->getDatabase());
$this->assertEquals("charset=utf8&timezone=UTC", $dsn->getQuery());
$dsn = new DSN("mariadb://user@localhost:3306/database?charset=utf8&timezone=UTC");
$this->assertEquals("mariadb", $dsn->getScheme());
$this->assertEquals("user", $dsn->getUser());
$this->assertNull($dsn->getPassword());
$this->assertEquals("localhost", $dsn->getHost());
$this->assertEquals("3306", $dsn->getPort());
$this->assertEquals("database", $dsn->getDatabase());
$this->assertEquals("charset=utf8&timezone=UTC", $dsn->getQuery());
$dsn = new DSN("mariadb://user@localhost/database?charset=utf8&timezone=UTC");
$this->assertEquals("mariadb", $dsn->getScheme());
$this->assertEquals("user", $dsn->getUser());
$this->assertNull($dsn->getPassword());
$this->assertEquals("localhost", $dsn->getHost());
$this->assertNull($dsn->getPort());
$this->assertEquals("database", $dsn->getDatabase());
$this->assertEquals("charset=utf8&timezone=UTC", $dsn->getQuery());
$dsn = new DSN("mariadb://user@localhost?charset=utf8&timezone=UTC");
$this->assertEquals("mariadb", $dsn->getScheme());
$this->assertEquals("user", $dsn->getUser());
$this->assertNull($dsn->getPassword());
$this->assertEquals("localhost", $dsn->getHost());
$this->assertNull($dsn->getPort());
$this->assertEmpty($dsn->getDatabase());
$this->assertEquals("charset=utf8&timezone=UTC", $dsn->getQuery());
$dsn = new DSN("mariadb://user@localhost");
$this->assertEquals("mariadb", $dsn->getScheme());
$this->assertEquals("user", $dsn->getUser());
$this->assertNull($dsn->getPassword());
$this->assertEquals("localhost", $dsn->getHost());
$this->assertNull($dsn->getPort());
$this->assertEmpty($dsn->getDatabase());
$this->assertNull($dsn->getQuery());
$dsn = new DSN("mariadb://user:@localhost");
$this->assertEquals("mariadb", $dsn->getScheme());
$this->assertEquals("user", $dsn->getUser());
$this->assertEmpty($dsn->getPassword());
$this->assertEquals("localhost", $dsn->getHost());
$this->assertNull($dsn->getPort());
$this->assertEmpty($dsn->getDatabase());
$this->assertNull($dsn->getQuery());
$dsn = new DSN("mariadb://localhost");
$this->assertEquals("mariadb", $dsn->getScheme());
$this->assertNull($dsn->getUser());
$this->assertNull($dsn->getPassword());
$this->assertEquals("localhost", $dsn->getHost());
$this->assertNull($dsn->getPort());
$this->assertEmpty($dsn->getDatabase());
$this->assertNull($dsn->getQuery());
}
public function testFail(): void
{
$this->expectException(\InvalidArgumentException::class);
new DSN("mariadb://");
}
}