From 0481a510a4621736b9878f8cadafcc57aa7484b8 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 13 Dec 2021 13:19:15 +0545 Subject: [PATCH 01/15] storage resource --- app/init.php | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/app/init.php b/app/init.php index f30583f5c8..2d5a03ae8b 100644 --- a/app/init.php +++ b/app/init.php @@ -51,6 +51,10 @@ use Swoole\Database\PDOPool; use Swoole\Database\RedisConfig; use Swoole\Database\RedisPool; use Utopia\Database\Query; +use Utopia\Storage\Storage; +use Utopia\Storage\Device\Local; +use Utopia\Storage\Device\S3; +use Utopia\Storage\Device\DoSpaces; const APP_NAME = 'Appwrite'; const APP_DOMAIN = 'appwrite.io'; @@ -821,6 +825,52 @@ App::setResource('dbForInternal', function($db, $cache, $project) { return $database; }, ['db', 'cache', 'project']); +App::setResource('storageSelf', function() { + return new Local(); +}); + +App::setResource('storageFiles', function($project) { + switch (App::getEnv('_APP_STORAGE_DEVICE', Storage::DEVICE_LOCAL)) { + case Storage::DEVICE_LOCAL:default: + return new Local(APP_STORAGE_UPLOADS . '/app-' . $project->getId()); + case Storage::DEVICE_S3: + $s3AccessKey = App::getEnv('_APP_STORAGE_DEVICE_S3_ACCESS_KEY', ''); + $s3SecretKey = App::getEnv('_APP_STORAGE_DEVICE_S3_SECRET', ''); + $s3Region = App::getEnv('_APP_STORAGE_DEVICE_S3_REGION', ''); + $s3Bucket = App::getEnv('_APP_STORAGE_DEVICE_S3_BUCKET', ''); + $s3Acl = 'private'; + return new S3(APP_STORAGE_UPLOADS . '/app-' . $project->getId(), $s3AccessKey, $s3SecretKey, $s3Bucket, $s3Region, $s3Acl); + case Storage::DEVICE_DO_SPACES: + $doSpacesAccessKey = App::getEnv('_APP_STORAGE_DEVICE_DO_SPACES_ACCESS_KEY', ''); + $doSpacesSecretKey = App::getEnv('_APP_STORAGE_DEVICE_DO_SPACES_SECRET', ''); + $doSpacesRegion = App::getEnv('_APP_STORAGE_DEVICE_DO_SPACES_REGION', ''); + $doSpacesBucket = App::getEnv('_APP_STORAGE_DEVICE_DO_SPACES_BUCKET', ''); + $doSpacesAcl = 'private'; + return new DoSpaces(APP_STORAGE_UPLOADS . '/app-' . $project->getId(), $doSpacesAccessKey, $doSpacesSecretKey, $doSpacesBucket, $doSpacesRegion, $doSpacesAcl); + } +}, ['project']); + +App::setResource('storageFunctions', function($project) { + switch (App::getEnv('_APP_STORAGE_DEVICE', Storage::DEVICE_LOCAL)) { + case Storage::DEVICE_LOCAL:default: + return new Local(APP_STORAGE_FUNCTIONS . '/app-' . $project->getId()); + case Storage::DEVICE_S3: + $s3AccessKey = App::getEnv('_APP_STORAGE_DEVICE_S3_ACCESS_KEY', ''); + $s3SecretKey = App::getEnv('_APP_STORAGE_DEVICE_S3_SECRET', ''); + $s3Region = App::getEnv('_APP_STORAGE_DEVICE_S3_REGION', ''); + $s3Bucket = App::getEnv('_APP_STORAGE_DEVICE_S3_BUCKET', ''); + $s3Acl = 'private'; + return new S3(APP_STORAGE_FUNCTIONS . '/app-' . $project->getId(), $s3AccessKey, $s3SecretKey, $s3Bucket, $s3Region, $s3Acl); + case Storage::DEVICE_DO_SPACES: + $doSpacesAccessKey = App::getEnv('_APP_STORAGE_DEVICE_DO_SPACES_ACCESS_KEY', ''); + $doSpacesSecretKey = App::getEnv('_APP_STORAGE_DEVICE_DO_SPACES_SECRET', ''); + $doSpacesRegion = App::getEnv('_APP_STORAGE_DEVICE_DO_SPACES_REGION', ''); + $doSpacesBucket = App::getEnv('_APP_STORAGE_DEVICE_DO_SPACES_BUCKET', ''); + $doSpacesAcl = 'private'; + return new DoSpaces(APP_STORAGE_FUNCTIONS . '/app-' . $project->getId(), $doSpacesAccessKey, $doSpacesSecretKey, $doSpacesBucket, $doSpacesRegion, $doSpacesAcl); + } +}, ['project']); + App::setResource('dbForExternal', function($db, $cache, $project) { $cache = new Cache(new RedisCache($cache)); From b25cd9274c89f031b33463829a0525ca7ed80d73 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 13 Dec 2021 13:53:17 +0545 Subject: [PATCH 02/15] storage device resource --- app/init.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/init.php b/app/init.php index 2d5a03ae8b..b67ea8ab0a 100644 --- a/app/init.php +++ b/app/init.php @@ -825,11 +825,11 @@ App::setResource('dbForInternal', function($db, $cache, $project) { return $database; }, ['db', 'cache', 'project']); -App::setResource('storageSelf', function() { +App::setResource('deviceLocal', function() { return new Local(); }); -App::setResource('storageFiles', function($project) { +App::setResource('deviceFiles', function($project) { switch (App::getEnv('_APP_STORAGE_DEVICE', Storage::DEVICE_LOCAL)) { case Storage::DEVICE_LOCAL:default: return new Local(APP_STORAGE_UPLOADS . '/app-' . $project->getId()); @@ -850,7 +850,7 @@ App::setResource('storageFiles', function($project) { } }, ['project']); -App::setResource('storageFunctions', function($project) { +App::setResource('deviceFunctions', function($project) { switch (App::getEnv('_APP_STORAGE_DEVICE', Storage::DEVICE_LOCAL)) { case Storage::DEVICE_LOCAL:default: return new Local(APP_STORAGE_FUNCTIONS . '/app-' . $project->getId()); From 80272c528f2fa19dc7214d1051ed4c34334d65de Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 13 Dec 2021 13:53:27 +0545 Subject: [PATCH 03/15] use storage from resource --- app/controllers/api/storage.php | 100 ++++++++++++++++---------------- 1 file changed, 49 insertions(+), 51 deletions(-) diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index 743df13b00..e22f3adc26 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -545,7 +545,9 @@ App::post('/v1/storage/buckets/:bucketId/files') ->inject('audits') ->inject('usage') ->inject('mode') - ->action(function ($bucketId, $fileId, $file, $read, $write, $request, $response, $dbForInternal, $dbForExternal, $user, $audits, $usage, $mode) { + ->inject('deviceFiles') + ->inject('deviceLocal') + ->action(function ($bucketId, $fileId, $file, $read, $write, $request, $response, $dbForInternal, $dbForExternal, $user, $audits, $usage, $mode, $deviceFiles, $deviceLocal) { /** @var Utopia\Swoole\Request $request */ /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ @@ -553,6 +555,8 @@ App::post('/v1/storage/buckets/:bucketId/files') /** @var Utopia\Database\Document $user */ /** @var Appwrite\Event\Event $audits */ /** @var Appwrite\Stats\Stats $usage */ + /** @var Utopia\Storage\Device $deviceFiles */ + /** @var Utopia\Storage\Device $deviceLocal */ /** @var string $mode */ $bucket = $dbForInternal->getDocument('buckets', $bucketId); @@ -629,17 +633,14 @@ App::post('/v1/storage/buckets/:bucketId/files') throw new Exception('File size not allowed', 400); } - $device = Storage::getDevice('files'); - $localDevice = Storage::getDevice('self'); - if (!$upload->isValid($fileTmpName)) { throw new Exception('Invalid file', 403); } // Save to storage - $fileSize ??= $localDevice->getFileSize($fileTmpName); - $path = $device->getPath($fileId . '.' . \pathinfo($fileName, PATHINFO_EXTENSION)); - $path = str_ireplace($device->getRoot(), $device->getRoot() . DIRECTORY_SEPARATOR . $bucket->getId(), $path); // Add bucket id to path after root + $fileSize ??= $deviceLocal->getFileSize($fileTmpName); + $path = $deviceFiles->getPath($fileId . '.' . \pathinfo($fileName, PATHINFO_EXTENSION)); + $path = str_ireplace($deviceFiles->getRoot(), $deviceFiles->getRoot() . DIRECTORY_SEPARATOR . $bucket->getId(), $path); // Add bucket id to path after root if($permissionBucket) { $file = Authorization::skip(function() use ($dbForExternal, $bucketId, $fileId) { @@ -649,7 +650,7 @@ App::post('/v1/storage/buckets/:bucketId/files') $file = $dbForExternal->getDocument('bucket_' . $bucketId, $fileId); } - $metadata = ['content_type' => $localDevice->getFileMimeType($fileTmpName)]; + $metadata = ['content_type' => $deviceLocal->getFileMimeType($fileTmpName)]; if (!$file->isEmpty()) { $chunks = $file->getAttribute('chunksTotal', 1); $metadata = $file->getAttribute('metadata', []); @@ -658,7 +659,7 @@ App::post('/v1/storage/buckets/:bucketId/files') } } - $chunksUploaded = $device->upload($fileTmpName, $path, $chunk, $chunks, $metadata); + $chunksUploaded = $deviceFiles->upload($fileTmpName, $path, $chunk, $chunks, $metadata); if (empty($chunksUploaded)) { throw new Exception('Failed uploading file', 500); } @@ -671,23 +672,23 @@ App::post('/v1/storage/buckets/:bucketId/files') (int) App::getEnv('_APP_STORAGE_ANTIVIRUS_PORT', 3310)); if (!$antiVirus->fileScan($path)) { - $device->delete($path); + $deviceFiles->delete($path); throw new Exception('Invalid file', 403); } } - $mimeType = $device->getFileMimeType($path); // Get mime-type before compression and encryption + $mimeType = $deviceFiles->getFileMimeType($path); // Get mime-type before compression and encryption $data = ''; // Compression if ($fileSize <= APP_STORAGE_READ_BUFFER) { - $data = $device->read($path); + $data = $deviceFiles->read($path); $compressor = new GZIP(); $data = $compressor->compress($data); } if ($bucket->getAttribute('encryption', true) && $fileSize <= APP_STORAGE_READ_BUFFER) { if(empty($data)) { - $data = $device->read($path); + $data = $deviceFiles->read($path); } $key = App::getEnv('_APP_OPENSSL_KEY_V1'); $iv = OpenSSL::randomPseudoBytes(OpenSSL::cipherIVLength(OpenSSL::CIPHER_AES_128_GCM)); @@ -695,15 +696,15 @@ App::post('/v1/storage/buckets/:bucketId/files') } if (!empty($data)) { - if (!$device->write($path, $data, $mimeType)) { + if (!$deviceFiles->write($path, $data, $mimeType)) { throw new Exception('Failed to save file', 500); } } - $sizeActual = $device->getFileSize($path); + $sizeActual = $deviceFiles->getFileSize($path); $algorithm = empty($compressor) ? '' : $compressor->getName(); - $fileHash = $device->getFileHash($path); + $fileHash = $deviceFiles->getFileHash($path); if ($bucket->getAttribute('encryption', true) && $fileSize <= APP_STORAGE_READ_BUFFER) { $openSSLVersion = '1'; @@ -1025,24 +1026,21 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview') ->inject('dbForExternal') ->inject('usage') ->inject('mode') - ->action(function ($bucketId, $fileId, $width, $height, $gravity, $quality, $borderWidth, $borderColor, $borderRadius, $opacity, $rotation, $background, $output, $request, $response, $project, $dbForInternal, $dbForExternal, $usage, $mode) { + ->inject('deviceFiles') + ->action(function ($bucketId, $fileId, $width, $height, $gravity, $quality, $borderWidth, $borderColor, $borderRadius, $opacity, $rotation, $background, $output, $request, $response, $project, $dbForInternal, $dbForExternal, $usage, $mode, $deviceFiles) { /** @var Utopia\Swoole\Request $request */ /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Document $project */ /** @var Utopia\Database\Database $dbForInternal */ /** @var Utopia\Database\Database $dbForExternal */ /** @var Appwrite\Stats\Stats $usage */ + /** @var Utopia\Storage\Device $deviceFiles */ /** @var string $mode */ - $storage = 'files'; - if (!\extension_loaded('imagick')) { throw new Exception('Imagick extension is missing', 500); } - if (!Storage::exists($storage)) { - throw new Exception('No such storage device', 400); - } $bucket = $dbForInternal->getDocument('buckets', $bucketId); if ($bucket->isEmpty() @@ -1067,7 +1065,7 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview') $fileLogos = Config::getParam('storage-logos'); $date = \date('D, d M Y H:i:s', \time() + (60 * 60 * 24 * 45)).' GMT'; // 45 days cache - $key = \md5($fileId.$width.$height.$gravity.$quality.$borderWidth.$borderColor.$borderRadius.$opacity.$rotation.$background.$storage.$output); + $key = \md5($fileId.$width.$height.$gravity.$quality.$borderWidth.$borderColor.$borderRadius.$opacity.$rotation.$background.$output); if ($bucket->getAttribute('permission')==='bucket') { // skip authorization @@ -1094,13 +1092,12 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview') $cipher = null; $background = (empty($background)) ? 'eceff1' : $background; $type = \strtolower(\pathinfo($path, PATHINFO_EXTENSION)); - $key = \md5($path.$width.$height.$gravity.$quality.$borderWidth.$borderColor.$borderRadius.$opacity.$rotation.$background.$storage.$output); + $key = \md5($path.$width.$height.$gravity.$quality.$borderWidth.$borderColor.$borderRadius.$opacity.$rotation.$background.$output); } $compressor = new GZIP(); - $device = Storage::getDevice('files'); - if (!$device->exists($path)) { + if (!$deviceFiles->exists($path)) { throw new Exception('File not found', 404); } @@ -1118,7 +1115,7 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview') ; } - $source = $device->read($path); + $source = $deviceFiles->read($path); if (!empty($cipher)) { // Decrypt $source = OpenSSL::decrypt( @@ -1200,12 +1197,14 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/download') ->inject('dbForExternal') ->inject('usage') ->inject('mode') - ->action(function ($bucketId, $fileId, $request, $response, $dbForInternal, $dbForExternal, $usage, $mode) { + ->inject('deviceFiles') + ->action(function ($bucketId, $fileId, $request, $response, $dbForInternal, $dbForExternal, $usage, $mode, $deviceFiles) { /** @var Utopia\Swoole\Request $request */ /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ /** @var Utopia\Database\Database $dbForExternal */ /** @var Appwrite\Stats\Stats $usage */ + /** @var Utopia\Storage\Device $deviceFiles */ /** @var string $mode */ $bucket = $dbForInternal->getDocument('buckets', $bucketId); @@ -1236,10 +1235,8 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/download') } $path = $file->getAttribute('path', ''); - - $device = Storage::getDevice('files'); - if (!$device->exists($path)) { + if (!$deviceFiles->exists($path)) { throw new Exception('File not found in ' . $path, 404); } @@ -1280,7 +1277,7 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/download') $source = ''; if (!empty($file->getAttribute('openSSLCipher'))) { // Decrypt - $source = $device->read($path); + $source = $deviceFiles->read($path); $source = OpenSSL::decrypt( $source, $file->getAttribute('openSSLCipher'), @@ -1293,7 +1290,7 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/download') if (!empty($file->getAttribute('algorithm', ''))) { if (empty($source)) { - $source = $device->read($path); + $source = $deviceFiles->read($path); } $compressor = new GZIP(); $source = $compressor->decompress($source); @@ -1307,14 +1304,14 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/download') } if (!empty($rangeHeader)) { - $response->send($device->read($path, $start, ($end - $start + 1))); + $response->send($deviceFiles->read($path, $start, ($end - $start + 1))); } if ($size > APP_STORAGE_READ_BUFFER) { - $response->addHeader('Content-Length', $device->getFileSize($path)); + $response->addHeader('Content-Length', $deviceFiles->getFileSize($path)); for ($i=0; $i < ceil($size / MAX_OUTPUT_CHUNK_SIZE); $i++) { $response->chunk( - $device->read( + $deviceFiles->read( $path, ($i * MAX_OUTPUT_CHUNK_SIZE), min(MAX_OUTPUT_CHUNK_SIZE, $size - ($i * MAX_OUTPUT_CHUNK_SIZE)) @@ -1323,7 +1320,7 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/download') ); } } else { - $response->send($device->read($path)); + $response->send($deviceFiles->read($path)); } }); @@ -1347,11 +1344,13 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/view') ->inject('dbForExternal') ->inject('usage') ->inject('mode') - ->action(function ($bucketId, $fileId, $response, $request, $dbForInternal, $dbForExternal, $usage, $mode) { + ->inject('deviceFiles') + ->action(function ($bucketId, $fileId, $response, $request, $dbForInternal, $dbForExternal, $usage, $mode, $deviceFiles) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Swoole\Request $request */ /** @var Utopia\Database\Database $dbForInternal */ /** @var Appwrite\Stats\Stats $usage */ + /** @var Utopia\Storage\Device $deviceFiles */ /** @var string $mode */ $bucket = $dbForInternal->getDocument('buckets', $bucketId); @@ -1385,8 +1384,7 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/view') $path = $file->getAttribute('path', ''); - $device = Storage::getDevice('files'); - if (!$device->exists($path)) { + if (!$deviceFiles->exists($path)) { throw new Exception('File not found in ' . $path, 404); } @@ -1432,7 +1430,7 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/view') $source = ''; if (!empty($file->getAttribute('openSSLCipher'))) { // Decrypt - $source = $device->read($path); + $source = $deviceFiles->read($path); $source = OpenSSL::decrypt( $source, $file->getAttribute('openSSLCipher'), @@ -1445,7 +1443,7 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/view') if (!empty($file->getAttribute('algorithm', ''))) { if (empty($source)) { - $source = $device->read($path); + $source = $deviceFiles->read($path); } $compressor = new GZIP(); $source = $compressor->decompress($source); @@ -1464,15 +1462,15 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/view') } if (!empty($rangeHeader)) { - $response->send($device->read($path, $start, ($end - $start + 1))); + $response->send($deviceFiles->read($path, $start, ($end - $start + 1))); } - $size = $device->getFileSize($path); + $size = $deviceFiles->getFileSize($path); if ($size > APP_STORAGE_READ_BUFFER) { - $response->addHeader('Content-Length', $device->getFileSize($path)); + $response->addHeader('Content-Length', $deviceFiles->getFileSize($path)); for ($i=0; $i < ceil($size / MAX_OUTPUT_CHUNK_SIZE); $i++) { $response->chunk( - $device->read( + $deviceFiles->read( $path, ($i * MAX_OUTPUT_CHUNK_SIZE), min(MAX_OUTPUT_CHUNK_SIZE, $size - ($i * MAX_OUTPUT_CHUNK_SIZE)) @@ -1481,7 +1479,7 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/view') ); } } else { - $response->send($device->read($path)); + $response->send($deviceFiles->read($path)); } }); @@ -1590,13 +1588,15 @@ App::delete('/v1/storage/buckets/:bucketId/files/:fileId') ->inject('audits') ->inject('usage') ->inject('mode') - ->action(function ($bucketId, $fileId, $response, $dbForInternal, $dbForExternal, $events, $audits, $usage, $mode) { + ->inject('deviceFiles') + ->action(function ($bucketId, $fileId, $response, $dbForInternal, $dbForExternal, $events, $audits, $usage, $mode, $deviceFiles) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ /** @var Utopia\Database\Database $dbForExternal */ /** @var Appwrite\Event\Event $events */ /** @var Appwrite\Event\Event $audits */ /** @var Appwrite\Stats\Stats $usage */ + /** @var Utopia\Storage\Device $deviceFiles */ /** @var string $mode */ $bucket = $dbForInternal->getDocument('buckets', $bucketId); @@ -1626,9 +1626,7 @@ App::delete('/v1/storage/buckets/:bucketId/files/:fileId') throw new Exception('File not found', 404); } - $device = Storage::getDevice('files'); - - if ($device->delete($file->getAttribute('path', ''))) { + if ($deviceFiles->delete($file->getAttribute('path', ''))) { if ($bucket->getAttribute('permission') === 'bucket') { $deleted = Authorization::skip(function() use ($dbForExternal, $fileId, $bucketId) { return $dbForExternal->deleteDocument('bucket_' . $bucketId, $fileId); From d9883fe9eb780508852d661372d4dd7bcc28fa79 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 13 Dec 2021 13:53:36 +0545 Subject: [PATCH 04/15] use storage from resources in functions --- app/controllers/api/functions.php | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index d665ed76b4..6cbddadb46 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -426,11 +426,15 @@ App::post('/v1/functions/:functionId/tags') ->inject('response') ->inject('dbForInternal') ->inject('usage') - ->action(function ($functionId, $command, $file, $request, $response, $dbForInternal, $usage) { + ->inject('deviceFunctions') + ->inject('deviceLocal') + ->action(function ($functionId, $command, $file, $request, $response, $dbForInternal, $usage, $deviceFunctions, $deviceLocal) { /** @var Utopia\Swoole\Request $request */ /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ /** @var Appwrite\Event\Event $usage */ + /** @var Utopia\Storage\Device $deviceFunctions */ + /** @var Utopia\Storage\Device $deviceLocal */ $function = $dbForInternal->getDocument('functions', $functionId); @@ -439,7 +443,6 @@ App::post('/v1/functions/:functionId/tags') } $file = $request->getFiles('code'); - $device = Storage::getDevice('functions'); $fileExt = new FileExt([FileExt::TYPE_GZIP]); $fileSizeValidator = new FileSize(App::getEnv('_APP_STORAGE_LIMIT', 0)); $upload = new Upload(); @@ -490,8 +493,8 @@ App::post('/v1/functions/:functionId/tags') } // Save to storage - $fileSize ??= Storage::getDevice('self')->getFileSize($fileTmpName); - $path = $device->getPath($tagId.'.'.\pathinfo($fileName, PATHINFO_EXTENSION)); + $fileSize ??= $deviceLocal->getFileSize($fileTmpName); + $path = $deviceFunctions->getPath($tagId.'.'.\pathinfo($fileName, PATHINFO_EXTENSION)); $tag = $dbForInternal->getDocument('tags', $tagId); @@ -502,14 +505,14 @@ App::post('/v1/functions/:functionId/tags') } } - $chunksUploaded = $device->upload($fileTmpName, $path, $chunk, $chunks); + $chunksUploaded = $deviceFunctions->upload($fileTmpName, $path, $chunk, $chunks); if (empty($chunksUploaded)) { throw new Exception('Failed moving file', 500); } if($chunksUploaded == $chunks) { - $fileSize = $device->getFileSize($path); + $fileSize = $deviceFunctions->getFileSize($path); if ($tag->isEmpty()) { $tag = $dbForInternal->createDocument('tags', new Document([ @@ -663,10 +666,12 @@ App::delete('/v1/functions/:functionId/tags/:tagId') ->inject('response') ->inject('dbForInternal') ->inject('usage') - ->action(function ($functionId, $tagId, $response, $dbForInternal, $usage) { + ->inject('deviceFunctions') + ->action(function ($functionId, $tagId, $response, $dbForInternal, $usage, $deviceFunctions) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ /** @var Appwrite\Event\Event $usage */ + /** @var Utopia\Storage\Device $deviceFunctions */ $function = $dbForInternal->getDocument('functions', $functionId); @@ -684,9 +689,7 @@ App::delete('/v1/functions/:functionId/tags/:tagId') throw new Exception('Tag not found', 404); } - $device = Storage::getDevice('functions'); - - if ($device->delete($tag->getAttribute('path', ''))) { + if ($deviceFunctions->delete($tag->getAttribute('path', ''))) { if (!$dbForInternal->deleteDocument('tags', $tag->getId())) { throw new Exception('Failed to remove tag from DB', 500); } From 350859d9bdc49f71b6ff5d08b329b6b1e5d7e233 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 13 Dec 2021 15:02:53 +0545 Subject: [PATCH 05/15] refactor files collection config --- app/config/sub_collections.php | 227 +++++++++++++++++++++++++++++ app/controllers/api/storage.php | 247 ++++---------------------------- app/init.php | 1 + 3 files changed, 259 insertions(+), 216 deletions(-) create mode 100644 app/config/sub_collections.php diff --git a/app/config/sub_collections.php b/app/config/sub_collections.php new file mode 100644 index 0000000000..332b68cd33 --- /dev/null +++ b/app/config/sub_collections.php @@ -0,0 +1,227 @@ + [ + 'attributes' => [ + [ + '$id' => 'dateCreated', + 'type' => Database::VAR_INTEGER, + 'format' => '', + 'size' => 0, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + 'array' => false, + '$id' => 'bucketId', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'name', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 2048, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'path', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 2048, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'signature', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 2048, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'mimeType', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 127, // https://tools.ietf.org/html/rfc4288#section-4.2 + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'metadata', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 16384, // https://tools.ietf.org/html/rfc4288#section-4.2 + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => ['json'], + ], + [ + '$id' => 'sizeOriginal', + 'type' => Database::VAR_INTEGER, + 'format' => '', + 'size' => 0, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'sizeActual', + 'type' => Database::VAR_INTEGER, + 'format' => '', + 'size' => 0, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'algorithm', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 255, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'comment', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 2048, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'openSSLVersion', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 64, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'openSSLCipher', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 64, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'openSSLTag', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 2048, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'openSSLIV', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 2048, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'chunksTotal', + 'type' => Database::VAR_INTEGER, + 'format' => '', + 'size' => 0, + 'signed' => false, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'chunksUploaded', + 'type' => Database::VAR_INTEGER, + 'format' => '', + 'size' => 0, + 'signed' => false, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'search', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 16384, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + ], + 'indexes' => [ + [ + '$id' => '_key_search', + 'type' => Database::INDEX_FULLTEXT, + 'attributes' => ['search'], + 'lengths' => [2048], + 'orders' => [Database::ORDER_ASC], + ], + [ + '$id' => '_key_bucket', + 'type' => Database::INDEX_KEY, + 'attributes' => ['bucketId'], + 'lengths' => [Database::LENGTH_KEY], + 'orders' => [Database::ORDER_ASC], + ], + ] + ], +]; + +return $subCollections; diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index e22f3adc26..37ed21a930 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -70,222 +70,37 @@ App::post('/v1/storage/buckets') $bucketId = $bucketId === 'unique()' ? $dbForInternal->getId() : $bucketId; try { - $dbForExternal->createCollection('bucket_' . $bucketId, [ - new Document([ - '$id' => 'dateCreated', - 'type' => Database::VAR_INTEGER, - 'format' => '', - 'size' => 0, - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ]), - new Document([ - 'array' => false, - '$id' => 'bucketId', - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => Database::LENGTH_KEY, - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ]), - new Document([ - '$id' => 'name', - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => 2048, - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ]), - new Document([ - '$id' => 'path', - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => 2048, - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ]), - new Document([ - '$id' => 'signature', - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => 2048, - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ]), - new Document([ - '$id' => 'mimeType', - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => 127, // https://tools.ietf.org/html/rfc4288#section-4.2 - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ]), - new Document([ - '$id' => 'metadata', - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => 16384, // https://tools.ietf.org/html/rfc4288#section-4.2 - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => ['json'], - ]), - new Document([ - '$id' => 'sizeOriginal', - 'type' => Database::VAR_INTEGER, - 'format' => '', - 'size' => 0, - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ]), - new Document([ - '$id' => 'sizeActual', - 'type' => Database::VAR_INTEGER, - 'format' => '', - 'size' => 0, - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ]), - new Document([ - '$id' => 'algorithm', - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => 255, - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ]), - new Document([ - '$id' => 'comment', - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => 2048, - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ]), - new Document([ - '$id' => 'openSSLVersion', - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => 64, - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ]), - new Document([ - '$id' => 'openSSLCipher', - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => 64, - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ]), - new Document([ - '$id' => 'openSSLTag', - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => 2048, - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ]), - new Document([ - '$id' => 'openSSLIV', - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => 2048, - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ]), - new Document([ - '$id' => 'chunksTotal', - 'type' => Database::VAR_INTEGER, - 'format' => '', - 'size' => 0, - 'signed' => false, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ]), - new Document([ - '$id' => 'chunksUploaded', - 'type' => Database::VAR_INTEGER, - 'format' => '', - 'size' => 0, - 'signed' => false, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ]), - new Document([ - '$id' => 'search', - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => 16384, - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ]), - ], [ - new Document([ - '$id' => '_key_search', - 'type' => Database::INDEX_FULLTEXT, - 'attributes' => ['search'], - 'lengths' => [2048], - 'orders' => [Database::ORDER_ASC], - ]), - new Document([ - '$id' => '_key_bucket', - 'type' => Database::INDEX_KEY, - 'attributes' => ['bucketId'], - 'lengths' => [Database::LENGTH_KEY], - 'orders' => [Database::ORDER_ASC], - ]), - ]); + $files = Config::getParam('collections', [])['files'] ?? []; + if(empty($files)) { + throw new Exception('Files collection is not configured.'); + } + + $attributes = []; + $indexes = []; + + foreach ($files['attributes'] as $attribute) { + $attributes[] = new Document([ + '$id' => $attribute['$id'], + 'type' => $attribute['type'], + 'size' => $attribute['size'], + 'required' => $attribute['required'], + 'signed' => $attribute['signed'], + 'array' => $attribute['array'], + 'filters' => $attribute['filters'], + ]); + } + + foreach ($files['indexes'] as $index) { + $indexes[] = new Document([ + '$id' => $index['$id'], + 'type' => $index['type'], + 'attributes' => $index['attributes'], + 'lengths' => $index['lengths'], + 'orders' => $index['orders'], + ]); + } + + $dbForExternal->createCollection('bucket_' . $bucketId, $attributes, $indexes); $bucket = $dbForInternal->createDocument('buckets', new Document([ '$id' => $bucketId, diff --git a/app/init.php b/app/init.php index b67ea8ab0a..672a8214fc 100644 --- a/app/init.php +++ b/app/init.php @@ -139,6 +139,7 @@ Config::load('auth', __DIR__.'/config/auth.php'); Config::load('providers', __DIR__.'/config/providers.php'); Config::load('platforms', __DIR__.'/config/platforms.php'); Config::load('collections', __DIR__.'/config/collections.php'); +Config::load('subCollections', __DIR__.'/config/sub_collections.php'); Config::load('runtimes', __DIR__.'/config/runtimes.php'); Config::load('roles', __DIR__.'/config/roles.php'); // User roles and scopes Config::load('scopes', __DIR__.'/config/scopes.php'); // User roles and scopes From 64c719e82c40a76415bb06c5ca79843489b2f23e Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 13 Dec 2021 17:31:59 +0545 Subject: [PATCH 06/15] re organize validators --- app/controllers/api/storage.php | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index 37ed21a930..0720482351 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -390,22 +390,12 @@ App::post('/v1/storage/buckets/:bucketId/files') } } - $file = $request->getFiles('file'); - - /** - * Validators - */ - $allowedFileExtensions = $bucket->getAttribute('allowedFileExtensions', []); - $fileExt = new FileExt($allowedFileExtensions); - $maximumFileSize = $bucket->getAttribute('maximumFileSize', 0); if ($maximumFileSize > (int) App::getEnv('_APP_STORAGE_LIMIT', 0)) { throw new Exception('Error bucket maximum file size is larger than _APP_STORAGE_LIMIT', 500); } - $fileSizeValidator = new FileSize($maximumFileSize); - $upload = new Upload(); - + $file = $request->getFiles('file'); if (empty($file)) { throw new Exception('No file sent', 400); } @@ -430,7 +420,7 @@ App::post('/v1/storage/buckets/:bucketId/files') } if ($end === $fileSize) { - //if it's a last chunks the chunk size might differ, so we set the $chunks and $chunk to notify it's last chunk + //if it's a last chunks the chunk size might differ, so we set the $chunks and $chunk to -1 notify it's last chunk $chunks = $chunk = -1; } else { // Calculate total number of chunks based on the chunk size i.e ($rangeEnd - $rangeStart) @@ -439,15 +429,23 @@ App::post('/v1/storage/buckets/:bucketId/files') } } - // Check if file type is allowed (feature for project settings?) + /** + * Validators + */ + // Check if file type is allowed + $allowedFileExtensions = $bucket->getAttribute('allowedFileExtensions', []); + $fileExt = new FileExt($allowedFileExtensions); if (!empty($allowedFileExtensions) && !$fileExt->isValid($fileName)) { throw new Exception('File extension not allowed', 400); } - if (!$fileSizeValidator->isValid($fileSize)) { // Check if file size is exceeding allowed limit + // Check if file size is exceeding allowed limit + $fileSizeValidator = new FileSize($maximumFileSize); + if (!$fileSizeValidator->isValid($fileSize)) { throw new Exception('File size not allowed', 400); } - + + $upload = new Upload(); if (!$upload->isValid($fileTmpName)) { throw new Exception('Invalid file', 403); } @@ -488,7 +486,7 @@ App::post('/v1/storage/buckets/:bucketId/files') if (!$antiVirus->fileScan($path)) { $deviceFiles->delete($path); - throw new Exception('Invalid file', 403); + throw new Exception('Invalid file', 400); } } From 0f8a3fcea37bba8549c59d4f3d42c936095c9a42 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 14 Dec 2021 10:57:48 +0545 Subject: [PATCH 07/15] fix collection config name --- app/controllers/api/storage.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index 0720482351..2ee823a3f0 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -70,7 +70,7 @@ App::post('/v1/storage/buckets') $bucketId = $bucketId === 'unique()' ? $dbForInternal->getId() : $bucketId; try { - $files = Config::getParam('collections', [])['files'] ?? []; + $files = Config::getParam('subCollections', [])['files'] ?? []; if(empty($files)) { throw new Exception('Files collection is not configured.'); } From 29a5de377c12b42bf725f16a4733c518e79db130 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 14 Dec 2021 12:34:54 +0545 Subject: [PATCH 08/15] fix custom id validator --- composer.lock | 68 ++++++++++---------- src/Appwrite/Database/Validator/CustomId.php | 2 +- 2 files changed, 35 insertions(+), 35 deletions(-) diff --git a/composer.lock b/composer.lock index 1eaf8b75b7..0bf24e53e2 100644 --- a/composer.lock +++ b/composer.lock @@ -2138,18 +2138,18 @@ }, { "name": "utopia-php/database", - "version": "0.12.0", + "version": "0.12.1", "source": { "type": "git", "url": "https://github.com/utopia-php/database", - "reference": "102ee1d21fd55fc92dc7a07b60672a98ae49be26" + "reference": "af512b7a00cc7c6e30fa03efbc5fd7e77a93e2df" }, "require": { "ext-mongodb": "*", "ext-pdo": "*", "ext-redis": "*", "mongodb/mongodb": "1.8.0", - "php": ">=7.1", + "php": ">=8.0", "utopia-php/cache": "0.4.*", "utopia-php/framework": "0.*.*" }, @@ -2191,7 +2191,7 @@ "upf", "utopia" ], - "time": "2021-11-24T14:53:22+00:00" + "time": "2021-12-13T14:57:32+00:00" }, { "name": "utopia-php/domains", @@ -2249,18 +2249,18 @@ }, { "name": "utopia-php/framework", - "version": "0.19.1", + "version": "0.19.2", "source": { "type": "git", "url": "https://github.com/utopia-php/framework", - "reference": "cc7629b5f7a8f45912ec2e069b7f14e361e41c34" + "reference": "49e4374b97c0f4d14bc84b424bdc9c3b7747e15f" }, "require": { - "php": ">=7.3.0" + "php": ">=8.0.0" }, "require-dev": { - "phpunit/phpunit": "^9.4", - "vimeo/psalm": "4.0.1" + "phpunit/phpunit": "^9.5.10", + "vimeo/psalm": "4.13.1" }, "type": "library", "autoload": { @@ -2283,7 +2283,7 @@ "php", "upf" ], - "time": "2021-11-25T16:11:40+00:00" + "time": "2021-12-07T09:29:35+00:00" }, { "name": "utopia-php/image", @@ -2554,7 +2554,7 @@ "source": { "type": "git", "url": "https://github.com/utopia-php/storage", - "reference": "c47611b3a4a36c674c16e9720e86187452eb1cc2" + "reference": "c558674fdda56161dd32cd9acdc7ced6741afa83" }, "require": { "php": ">=8.0", @@ -2592,31 +2592,31 @@ "upf", "utopia" ], - "time": "2021-12-09T07:34:55+00:00" + "time": "2021-12-09T08:20:04+00:00" }, { "name": "utopia-php/swoole", - "version": "0.3.1", + "version": "0.3.2", "source": { "type": "git", "url": "https://github.com/utopia-php/swoole.git", - "reference": "b564dacb13472845f06df158ae5382e8098dfd7a" + "reference": "2b714eddf77cd5eda1889219c9656d7c0a63ce73" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/swoole/zipball/b564dacb13472845f06df158ae5382e8098dfd7a", - "reference": "b564dacb13472845f06df158ae5382e8098dfd7a", + "url": "https://api.github.com/repos/utopia-php/swoole/zipball/2b714eddf77cd5eda1889219c9656d7c0a63ce73", + "reference": "2b714eddf77cd5eda1889219c9656d7c0a63ce73", "shasum": "" }, "require": { "ext-swoole": "*", - "php": ">=7.4", + "php": ">=8.0", "utopia-php/framework": "0.*.*" }, "require-dev": { "phpunit/phpunit": "^9.3", - "swoole/ide-helper": "4.5.5", - "vimeo/psalm": "4.0.1" + "swoole/ide-helper": "4.8.3", + "vimeo/psalm": "4.15.0" }, "type": "library", "autoload": { @@ -2646,9 +2646,9 @@ ], "support": { "issues": "https://github.com/utopia-php/swoole/issues", - "source": "https://github.com/utopia-php/swoole/tree/0.3.1" + "source": "https://github.com/utopia-php/swoole/tree/0.3.2" }, - "time": "2021-07-27T09:28:10+00:00" + "time": "2021-12-13T15:37:41+00:00" }, { "name": "utopia-php/system", @@ -5642,23 +5642,23 @@ }, { "name": "symfony/console", - "version": "v5.4.0", + "version": "v5.4.1", "source": { "type": "git", "url": "https://github.com/symfony/console.git", - "reference": "ec3661faca1d110d6c307e124b44f99ac54179e3" + "reference": "9130e1a0fc93cb0faadca4ee917171bd2ca9e5f4" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/console/zipball/ec3661faca1d110d6c307e124b44f99ac54179e3", - "reference": "ec3661faca1d110d6c307e124b44f99ac54179e3", + "url": "https://api.github.com/repos/symfony/console/zipball/9130e1a0fc93cb0faadca4ee917171bd2ca9e5f4", + "reference": "9130e1a0fc93cb0faadca4ee917171bd2ca9e5f4", "shasum": "" }, "require": { "php": ">=7.2.5", "symfony/deprecation-contracts": "^2.1|^3", "symfony/polyfill-mbstring": "~1.0", - "symfony/polyfill-php73": "^1.8", + "symfony/polyfill-php73": "^1.9", "symfony/polyfill-php80": "^1.16", "symfony/service-contracts": "^1.1|^2|^3", "symfony/string": "^5.1|^6.0" @@ -5721,7 +5721,7 @@ "terminal" ], "support": { - "source": "https://github.com/symfony/console/tree/v5.4.0" + "source": "https://github.com/symfony/console/tree/v5.4.1" }, "funding": [ { @@ -5737,7 +5737,7 @@ "type": "tidelift" } ], - "time": "2021-11-29T15:30:56+00:00" + "time": "2021-12-09T11:22:43+00:00" }, { "name": "symfony/polyfill-intl-grapheme", @@ -6147,16 +6147,16 @@ }, { "name": "symfony/string", - "version": "v6.0.0", + "version": "v6.0.1", "source": { "type": "git", "url": "https://github.com/symfony/string.git", - "reference": "ba727797426af0f587f4800566300bdc0cda0777" + "reference": "0cfed595758ec6e0a25591bdc8ca733c1896af32" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/string/zipball/ba727797426af0f587f4800566300bdc0cda0777", - "reference": "ba727797426af0f587f4800566300bdc0cda0777", + "url": "https://api.github.com/repos/symfony/string/zipball/0cfed595758ec6e0a25591bdc8ca733c1896af32", + "reference": "0cfed595758ec6e0a25591bdc8ca733c1896af32", "shasum": "" }, "require": { @@ -6212,7 +6212,7 @@ "utf8" ], "support": { - "source": "https://github.com/symfony/string/tree/v6.0.0" + "source": "https://github.com/symfony/string/tree/v6.0.1" }, "funding": [ { @@ -6228,7 +6228,7 @@ "type": "tidelift" } ], - "time": "2021-10-29T07:35:21+00:00" + "time": "2021-12-08T15:13:44+00:00" }, { "name": "textalk/websocket", diff --git a/src/Appwrite/Database/Validator/CustomId.php b/src/Appwrite/Database/Validator/CustomId.php index 59c14de8be..39a8500f81 100644 --- a/src/Appwrite/Database/Validator/CustomId.php +++ b/src/Appwrite/Database/Validator/CustomId.php @@ -13,7 +13,7 @@ class CustomId extends Key { * * @return bool */ - public function isValid($value) + public function isValid($value): bool { return $value == 'unique()' || parent::isValid($value); From 535bdee74af0245bb584fdafe902972a129e6119 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 23 Dec 2021 14:16:32 +0545 Subject: [PATCH 09/15] update remove sub collection and use the configured collections itself --- app/config/collections.php | 233 ++++++++++++++++++++++++++++++- app/config/sub_collections.php | 227 ------------------------------ app/controllers/api/projects.php | 3 + app/controllers/api/storage.php | 2 +- app/http.php | 4 + app/init.php | 1 - 6 files changed, 239 insertions(+), 231 deletions(-) delete mode 100644 app/config/sub_collections.php diff --git a/app/config/collections.php b/app/config/collections.php index 396e297ca1..85d8cb5dd0 100644 --- a/app/config/collections.php +++ b/app/config/collections.php @@ -6,6 +6,15 @@ use Utopia\Database\Database; $providers = Config::getParam('providers', []); $auth = Config::getParam('auth', []); +/** + * $collection => id of the parent collection where this will be inserted + * $id => id of this collection + * name => name of this collection + * project => whether or not this collection should be created per project + * attributes => list of attributes + * indexes => list of indexes + */ + $collections = [ 'collections' => [ '$collection' => Database::METADATA, @@ -2045,9 +2054,7 @@ $collections = [ 'buckets' => [ '$collection' => Database::METADATA, '$id' => 'buckets', - '$permissions' => ['read' => ['*']], 'name' => 'Buckets', - 'structure' => true, 'attributes' => [ [ '$id' => 'dateCreated', @@ -2324,6 +2331,228 @@ $collections = [ ], ] ], + 'files' => [ + '$collection' => 'buckets', + '$id' => 'files', + '$name' => 'Files', + 'attributes' => [ + [ + '$id' => 'dateCreated', + 'type' => Database::VAR_INTEGER, + 'format' => '', + 'size' => 0, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + 'array' => false, + '$id' => 'bucketId', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'name', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 2048, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'path', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 2048, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'signature', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 2048, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'mimeType', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 127, // https://tools.ietf.org/html/rfc4288#section-4.2 + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'metadata', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 16384, // https://tools.ietf.org/html/rfc4288#section-4.2 + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => ['json'], + ], + [ + '$id' => 'sizeOriginal', + 'type' => Database::VAR_INTEGER, + 'format' => '', + 'size' => 0, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'sizeActual', + 'type' => Database::VAR_INTEGER, + 'format' => '', + 'size' => 0, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'algorithm', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 255, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'comment', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 2048, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'openSSLVersion', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 64, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'openSSLCipher', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 64, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'openSSLTag', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 2048, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'openSSLIV', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 2048, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'chunksTotal', + 'type' => Database::VAR_INTEGER, + 'format' => '', + 'size' => 0, + 'signed' => false, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'chunksUploaded', + 'type' => Database::VAR_INTEGER, + 'format' => '', + 'size' => 0, + 'signed' => false, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'search', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 16384, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + ], + 'indexes' => [ + [ + '$id' => '_key_search', + 'type' => Database::INDEX_FULLTEXT, + 'attributes' => ['search'], + 'lengths' => [2048], + 'orders' => [Database::ORDER_ASC], + ], + [ + '$id' => '_key_bucket', + 'type' => Database::INDEX_KEY, + 'attributes' => ['bucketId'], + 'lengths' => [Database::LENGTH_KEY], + 'orders' => [Database::ORDER_ASC], + ], + ] + ], ]; return $collections; diff --git a/app/config/sub_collections.php b/app/config/sub_collections.php deleted file mode 100644 index 332b68cd33..0000000000 --- a/app/config/sub_collections.php +++ /dev/null @@ -1,227 +0,0 @@ - [ - 'attributes' => [ - [ - '$id' => 'dateCreated', - 'type' => Database::VAR_INTEGER, - 'format' => '', - 'size' => 0, - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ], - [ - 'array' => false, - '$id' => 'bucketId', - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => Database::LENGTH_KEY, - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ], - [ - '$id' => 'name', - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => 2048, - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ], - [ - '$id' => 'path', - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => 2048, - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ], - [ - '$id' => 'signature', - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => 2048, - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ], - [ - '$id' => 'mimeType', - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => 127, // https://tools.ietf.org/html/rfc4288#section-4.2 - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ], - [ - '$id' => 'metadata', - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => 16384, // https://tools.ietf.org/html/rfc4288#section-4.2 - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => ['json'], - ], - [ - '$id' => 'sizeOriginal', - 'type' => Database::VAR_INTEGER, - 'format' => '', - 'size' => 0, - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ], - [ - '$id' => 'sizeActual', - 'type' => Database::VAR_INTEGER, - 'format' => '', - 'size' => 0, - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ], - [ - '$id' => 'algorithm', - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => 255, - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ], - [ - '$id' => 'comment', - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => 2048, - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ], - [ - '$id' => 'openSSLVersion', - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => 64, - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ], - [ - '$id' => 'openSSLCipher', - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => 64, - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ], - [ - '$id' => 'openSSLTag', - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => 2048, - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ], - [ - '$id' => 'openSSLIV', - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => 2048, - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ], - [ - '$id' => 'chunksTotal', - 'type' => Database::VAR_INTEGER, - 'format' => '', - 'size' => 0, - 'signed' => false, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ], - [ - '$id' => 'chunksUploaded', - 'type' => Database::VAR_INTEGER, - 'format' => '', - 'size' => 0, - 'signed' => false, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ], - [ - '$id' => 'search', - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => 16384, - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ], - ], - 'indexes' => [ - [ - '$id' => '_key_search', - 'type' => Database::INDEX_FULLTEXT, - 'attributes' => ['search'], - 'lengths' => [2048], - 'orders' => [Database::ORDER_ASC], - ], - [ - '$id' => '_key_bucket', - 'type' => Database::INDEX_KEY, - 'attributes' => ['bucketId'], - 'lengths' => [Database::LENGTH_KEY], - 'orders' => [Database::ORDER_ASC], - ], - ] - ], -]; - -return $subCollections; diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index 53c07ef27d..ee16f15702 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -118,6 +118,9 @@ App::post('/v1/projects') $adapter->setup(); foreach ($collections as $key => $collection) { + if(($collection['$collection'] ?? '') !== Database::METADATA) { + continue; + } $attributes = []; $indexes = []; diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index 41c5c87ae9..951897eadf 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -71,7 +71,7 @@ App::post('/v1/storage/buckets') $bucketId = $bucketId === 'unique()' ? $dbForInternal->getId() : $bucketId; try { - $files = Config::getParam('subCollections', [])['files'] ?? []; + $files = Config::getParam('collections', [])['files'] ?? []; if(empty($files)) { throw new Exception('Files collection is not configured.'); } diff --git a/app/http.php b/app/http.php index e9d75c360a..dc27e28f06 100644 --- a/app/http.php +++ b/app/http.php @@ -13,6 +13,7 @@ use Utopia\Config\Config; use Utopia\Database\Validator\Authorization; use Utopia\Audit\Audit; use Utopia\Abuse\Adapters\TimeLimit; +use Utopia\Database\Database; use Utopia\Database\Document; use Utopia\Swoole\Files; use Utopia\Swoole\Request; @@ -94,6 +95,9 @@ $http->on('start', function (Server $http) use ($payloadSize, $register) { $adapter->setup(); foreach ($collections as $key => $collection) { + if(($collection['$collection'] ?? '') !== Database::METADATA) { + continue; + } Console::success('[Setup] - Creating collection: ' . $collection['$id'] . '...'); $attributes = []; diff --git a/app/init.php b/app/init.php index 2ece574e47..daaaae3551 100644 --- a/app/init.php +++ b/app/init.php @@ -136,7 +136,6 @@ Config::load('auth', __DIR__.'/config/auth.php'); Config::load('providers', __DIR__.'/config/providers.php'); Config::load('platforms', __DIR__.'/config/platforms.php'); Config::load('collections', __DIR__.'/config/collections.php'); -Config::load('subCollections', __DIR__.'/config/sub_collections.php'); Config::load('runtimes', __DIR__.'/config/runtimes.php'); Config::load('roles', __DIR__.'/config/roles.php'); // User roles and scopes Config::load('scopes', __DIR__.'/config/scopes.php'); // User roles and scopes From b6d0607c72cfe9adc1127666e462f19f21b99133 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 23 Dec 2021 14:43:11 +0545 Subject: [PATCH 10/15] update console db for default buckets --- app/http.php | 54 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/app/http.php b/app/http.php index dc27e28f06..f8d7ced6f2 100644 --- a/app/http.php +++ b/app/http.php @@ -128,6 +128,60 @@ $http->on('start', function (Server $http) use ($payloadSize, $register) { $dbForConsole->createCollection($key, $attributes, $indexes); } + Console::success('[Setup] - Creating default bucket...'); + $dbForConsole->createDocument('buckets', new Document([ + '$id' => 'default', + '$collection' => 'buckets', + 'dateCreated' => \time(), + 'dateUpdated' => \time(), + 'name' => 'Default', + 'permission' => 'file', + 'maximumFileSize' => App::getEnv('_APP_STORAGE_LIMIT', 0), // 10MB + 'allowedFileExtensions' => [], + 'enabled' => true, + 'adapter' => '', + 'encryption' => true, + 'antiVirus' => true, + '$read' => [], + '$write' => [], + 'search' => 'buckets Default', + ])); + + Console::success('[Setup] - Creating files collection for default bucket...'); + $files = $collections['file'] ?? []; + if(empty($files)) { + throw new Exception('Files collection is not configured.'); + } + + $attributes = []; + $indexes = []; + + foreach ($files['attributes'] as $attribute) { + $attributes[] = new Document([ + '$id' => $attribute['$id'], + 'type' => $attribute['type'], + 'size' => $attribute['size'], + 'required' => $attribute['required'], + 'signed' => $attribute['signed'], + 'array' => $attribute['array'], + 'filters' => $attribute['filters'], + ]); + } + + foreach ($files['indexes'] as $index) { + $indexes[] = new Document([ + '$id' => $index['$id'], + 'type' => $index['type'], + 'attributes' => $index['attributes'], + 'lengths' => $index['lengths'], + 'orders' => $index['orders'], + ]); + } + + $dbForExternal = $app->getResource('dbForExternal'); /** @var Utopia\Database\Database $dbForExternal */ + + $dbForExternal->createCollection('bucket_' . 'default', $attributes, $indexes); + Console::success('[Setup] - Server database init completed...'); } }); From 68a83fa5297d22b0110ac07a1806a63803ee0c88 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 23 Dec 2021 15:18:38 +0545 Subject: [PATCH 11/15] console external db --- app/init.php | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/app/init.php b/app/init.php index daaaae3551..7b06f0b404 100644 --- a/app/init.php +++ b/app/init.php @@ -850,6 +850,15 @@ App::setResource('dbForConsole', function($db, $cache) { return $database; }, ['db', 'cache']); +App::setResource('dbForConsoleExternal', function($db, $cache) { + $cache = new Cache(new RedisCache($cache)); + + $database = new Database(new MariaDB($db), $cache); + $database->setNamespace('project_console_external'); + + return $database; +}, ['db', 'cache']); + App::setResource('mode', function($request) { /** @var Utopia\Swoole\Request $request */ From 4cd8f05f69130aff6f48253fe07a3f1fad9fe11b Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 23 Dec 2021 15:18:44 +0545 Subject: [PATCH 12/15] fixes to files collection --- app/http.php | 73 +++++++++++++++------------- public/scripts/views/forms/upload.js | 2 +- 2 files changed, 39 insertions(+), 36 deletions(-) diff --git a/app/http.php b/app/http.php index f8d7ced6f2..ca289b615f 100644 --- a/app/http.php +++ b/app/http.php @@ -136,52 +136,55 @@ $http->on('start', function (Server $http) use ($payloadSize, $register) { 'dateUpdated' => \time(), 'name' => 'Default', 'permission' => 'file', - 'maximumFileSize' => App::getEnv('_APP_STORAGE_LIMIT', 0), // 10MB + 'maximumFileSize' => (int) App::getEnv('_APP_STORAGE_LIMIT', 0), // 10MB 'allowedFileExtensions' => [], 'enabled' => true, 'adapter' => '', 'encryption' => true, 'antiVirus' => true, '$read' => [], - '$write' => [], + '$write' => ['role:all'], 'search' => 'buckets Default', ])); - Console::success('[Setup] - Creating files collection for default bucket...'); - $files = $collections['file'] ?? []; - if(empty($files)) { - throw new Exception('Files collection is not configured.'); + $dbForConsoleExternal = $app->getResource('dbForConsoleExternal'); /** @var Utopia\Database\Database $dbForConsoleExternal */ + + if(!$dbForConsoleExternal->exists()) { + $dbForConsoleExternal->create(); + Console::success('[Setup] - Creating files collection for default bucket...'); + $files = $collections['files'] ?? []; + if(empty($files)) { + throw new Exception('Files collection is not configured.'); + } + + $attributes = []; + $indexes = []; + + foreach ($files['attributes'] as $attribute) { + $attributes[] = new Document([ + '$id' => $attribute['$id'], + 'type' => $attribute['type'], + 'size' => $attribute['size'], + 'required' => $attribute['required'], + 'signed' => $attribute['signed'], + 'array' => $attribute['array'], + 'filters' => $attribute['filters'], + ]); + } + + foreach ($files['indexes'] as $index) { + $indexes[] = new Document([ + '$id' => $index['$id'], + 'type' => $index['type'], + 'attributes' => $index['attributes'], + 'lengths' => $index['lengths'], + 'orders' => $index['orders'], + ]); + } + + $dbForConsoleExternal->createCollection('bucket_' . 'default', $attributes, $indexes); } - $attributes = []; - $indexes = []; - - foreach ($files['attributes'] as $attribute) { - $attributes[] = new Document([ - '$id' => $attribute['$id'], - 'type' => $attribute['type'], - 'size' => $attribute['size'], - 'required' => $attribute['required'], - 'signed' => $attribute['signed'], - 'array' => $attribute['array'], - 'filters' => $attribute['filters'], - ]); - } - - foreach ($files['indexes'] as $index) { - $indexes[] = new Document([ - '$id' => $index['$id'], - 'type' => $index['type'], - 'attributes' => $index['attributes'], - 'lengths' => $index['lengths'], - 'orders' => $index['orders'], - ]); - } - - $dbForExternal = $app->getResource('dbForExternal'); /** @var Utopia\Database\Database $dbForExternal */ - - $dbForExternal->createCollection('bucket_' . 'default', $attributes, $indexes); - Console::success('[Setup] - Server database init completed...'); } }); diff --git a/public/scripts/views/forms/upload.js b/public/scripts/views/forms/upload.js index 9fdd1bda66..472176eb28 100644 --- a/public/scripts/views/forms/upload.js +++ b/public/scripts/views/forms/upload.js @@ -108,7 +108,7 @@ expression.parse(element.dataset["write"] || "[]") ); - sdk.storage.createFile('unique()', files[0], read, write, 1).then( + sdk.storage.createFile('default', 'unique()', files[0], read, write, 1).then( function(response) { onComplete(message); From 36409dafc68bf1707bfa265eff0eeff1efcfe130 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 23 Dec 2021 15:53:37 +0545 Subject: [PATCH 13/15] fix bucket role --- app/http.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/http.php b/app/http.php index ca289b615f..7c0ed9fbf2 100644 --- a/app/http.php +++ b/app/http.php @@ -142,7 +142,7 @@ $http->on('start', function (Server $http) use ($payloadSize, $register) { 'adapter' => '', 'encryption' => true, 'antiVirus' => true, - '$read' => [], + '$read' => ['role:all'], '$write' => ['role:all'], 'search' => 'buckets Default', ])); From a790edf8d702cb23d297b4fce4490a3a3812c45c Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Fri, 7 Jan 2022 17:14:52 +0545 Subject: [PATCH 14/15] setting file size attributes to bigint --- app/config/collections.php | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/app/config/collections.php b/app/config/collections.php index a5cc87d181..2789501cf2 100644 --- a/app/config/collections.php +++ b/app/config/collections.php @@ -2071,7 +2071,7 @@ $collections = [ 'type' => Database::VAR_INTEGER, 'size' => 0, 'format' => '', - 'signed' => true, + 'signed' => false, 'required' => false, 'array' => false, 'filters' => [], @@ -2130,8 +2130,8 @@ $collections = [ [ '$id' => 'maximumFileSize', 'type' => Database::VAR_INTEGER, - 'signed' => true, - 'size' => 0, + 'signed' => false, + 'size' => 8, 'format' => '', 'filters' => [], 'required' => true, @@ -2341,7 +2341,7 @@ $collections = [ 'type' => Database::VAR_INTEGER, 'format' => '', 'size' => 0, - 'signed' => true, + 'signed' => false, 'required' => false, 'default' => null, 'array' => false, @@ -2418,8 +2418,8 @@ $collections = [ '$id' => 'sizeOriginal', 'type' => Database::VAR_INTEGER, 'format' => '', - 'size' => 0, - 'signed' => true, + 'size' => 8, + 'signed' => false, 'required' => false, 'default' => null, 'array' => false, @@ -2429,8 +2429,8 @@ $collections = [ '$id' => 'sizeActual', 'type' => Database::VAR_INTEGER, 'format' => '', - 'size' => 0, - 'signed' => true, + 'size' => 8, + 'signed' => false, 'required' => false, 'default' => null, 'array' => false, From e16f64d84bcf76ffe18f0604264ec46c5e0b61a0 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 10 Jan 2022 12:16:33 +0545 Subject: [PATCH 15/15] fix default bucket creation issue --- app/http.php | 80 +++++++++++++++++++++++++++------------------------- 1 file changed, 41 insertions(+), 39 deletions(-) diff --git a/app/http.php b/app/http.php index 8a43fce8f0..cc71dd9a98 100644 --- a/app/http.php +++ b/app/http.php @@ -109,43 +109,44 @@ $http->on('start', function (Server $http) use ($payloadSize, $register) { $adapter->setup(); } - foreach ($collections as $key => $collection) { - if(($collection['$collection'] ?? '') !== Database::METADATA) { - continue; - } - if(!$dbForConsole->getCollection($key)->isEmpty()) { - continue; - } - Console::success('[Setup] - Creating collection: ' . $collection['$id'] . '...'); + foreach ($collections as $key => $collection) { + if(($collection['$collection'] ?? '') !== Database::METADATA) { + continue; + } + if(!$dbForConsole->getCollection($key)->isEmpty()) { + continue; + } + Console::success('[Setup] - Creating collection: ' . $collection['$id'] . '...'); - $attributes = []; - $indexes = []; + $attributes = []; + $indexes = []; - foreach ($collection['attributes'] as $attribute) { - $attributes[] = new Document([ - '$id' => $attribute['$id'], - 'type' => $attribute['type'], - 'size' => $attribute['size'], - 'required' => $attribute['required'], - 'signed' => $attribute['signed'], - 'array' => $attribute['array'], - 'filters' => $attribute['filters'], - ]); - } - - foreach ($collection['indexes'] as $index) { - $indexes[] = new Document([ - '$id' => $index['$id'], - 'type' => $index['type'], - 'attributes' => $index['attributes'], - 'lengths' => $index['lengths'], - 'orders' => $index['orders'], - ]); - } - - $dbForConsole->createCollection($key, $attributes, $indexes); + foreach ($collection['attributes'] as $attribute) { + $attributes[] = new Document([ + '$id' => $attribute['$id'], + 'type' => $attribute['type'], + 'size' => $attribute['size'], + 'required' => $attribute['required'], + 'signed' => $attribute['signed'], + 'array' => $attribute['array'], + 'filters' => $attribute['filters'], + ]); } + foreach ($collection['indexes'] as $index) { + $indexes[] = new Document([ + '$id' => $index['$id'], + 'type' => $index['type'], + 'attributes' => $index['attributes'], + 'lengths' => $index['lengths'], + 'orders' => $index['orders'], + ]); + } + + $dbForConsole->createCollection($key, $attributes, $indexes); + } + + if($dbForConsole->getDocument('buckets', 'default')->isEmpty()) { Console::success('[Setup] - Creating default bucket...'); $dbForConsole->createDocument('buckets', new Document([ '$id' => 'default', @@ -164,16 +165,16 @@ $http->on('start', function (Server $http) use ($payloadSize, $register) { '$write' => ['role:all'], 'search' => 'buckets Default', ])); - + Console::success('[Setup] - Creating files collection for default bucket...'); $files = $collections['files'] ?? []; if(empty($files)) { throw new Exception('Files collection is not configured.'); } - + $attributes = []; $indexes = []; - + foreach ($files['attributes'] as $attribute) { $attributes[] = new Document([ '$id' => $attribute['$id'], @@ -185,7 +186,7 @@ $http->on('start', function (Server $http) use ($payloadSize, $register) { 'filters' => $attribute['filters'], ]); } - + foreach ($files['indexes'] as $index) { $indexes[] = new Document([ '$id' => $index['$id'], @@ -195,10 +196,11 @@ $http->on('start', function (Server $http) use ($payloadSize, $register) { 'orders' => $index['orders'], ]); } - + $dbForConsole->createCollection('bucket_' . 'default', $attributes, $indexes); + } - Console::success('[Setup] - Server database init completed...'); + Console::success('[Setup] - Server database init completed...'); }); Console::success('Server started successfully (max payload is '.number_format($payloadSize).' bytes)');