From 5db04e09f9e3697755a31629ae0c8c7898beae06 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Wed, 7 Jul 2021 15:52:11 +0545 Subject: [PATCH] first working POC of large file upload --- app/config/collections2.php | 20 ++ app/controllers/api/storage.php | 373 +++++++++++++------- composer.json | 6 +- composer.lock | 24 +- src/Appwrite/Utopia/Response/Model/File.php | 12 + tests/e2e/Services/Storage/StorageBase.php | 43 +++ 6 files changed, 324 insertions(+), 154 deletions(-) diff --git a/app/config/collections2.php b/app/config/collections2.php index c0cea304e2..56c068715e 100644 --- a/app/config/collections2.php +++ b/app/config/collections2.php @@ -625,6 +625,26 @@ $collections = [ 'array' => false, 'filters' => [], ], + [ + '$id' => 'totalChunks', + 'type' => Database::VAR_INTEGER, + 'format' => '', + 'size' => 0, + 'signed' => false, + 'required' => false, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'uploadedChunks', + 'type' => Database::VAR_INTEGER, + 'format' => '', + 'size' => 0, + 'signed' => false, + 'required' => false, + 'array' => false, + 'filters' => [], + ], ], 'indexes' => [ [ diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index eec8a6b766..16dd37622a 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -1,30 +1,30 @@ desc('Create storage bucket') @@ -41,7 +41,7 @@ App::post('/v1/storage/buckets') ->param('name', '', new Text(128), 'Bucket name', false) ->param('read', [], new ArrayList(new Text(64)), 'An array of strings with read permissions. By default no user is granted with any read permissions. [learn more about permissions](/docs/permissions) and get a full list of available permissions.', true) ->param('write', [], new ArrayList(new Text(64)), 'An array of strings with write permissions. By default no user is granted with any write permissions. [learn more about permissions](/docs/permissions) and get a full list of available permissions.', true) - ->param('maximumFileSize', (int) App::getEnv('_APP_STORAGE_LIMIT', 0) , new Integer(), 'Maximum file size allowed in bytes. Maximum allowed value is ' . App::getEnv('_APP_STORAGE_LIMIT', 0) . '. For self-hosted setups you can change the max limit by changing the `_APP_STORAGE_LIMIT` environment variable. [Learn more about storage environment variables](docs/environment-variables#storage)', true) + ->param('maximumFileSize', (int) App::getEnv('_APP_STORAGE_LIMIT', 0), new Integer(), 'Maximum file size allowed in bytes. Maximum allowed value is ' . App::getEnv('_APP_STORAGE_LIMIT', 0) . '. For self-hosted setups you can change the max limit by changing the `_APP_STORAGE_LIMIT` environment variable. [Learn more about storage environment variables](docs/environment-variables#storage)', true) ->param('allowedFileExtensions', [], new ArrayList(new Text(64)), 'Allowed file extensions', true) ->param('enabled', true, new Boolean(), 'Is bucket enabled?', true) ->param('adapter', 'local', new WhiteList(['local']), 'Storage adapter.', true) @@ -171,18 +171,18 @@ App::put('/v1/storage/buckets/:bucketId') throw new Exception('Bucket not found', 404); } - $read ??= $bucket->getAttribute('$read', []); // By default inherit read permissions - $write ??= $bucket->getAttribute('$write',[]); // By default inherit write permissions + $read??=$bucket->getAttribute('$read', []); // By default inherit read permissions + $write??=$bucket->getAttribute('$write', []); // By default inherit write permissions $bucket = $dbForInternal->updateDocument('buckets', $bucket->getId(), $bucket - ->setAttribute('name',$name) - ->setAttribute('$read',$read) - ->setAttribute('$write',$write) - ->setAttribute('maximumFileSize',$maximumFileSize) - ->setAttribute('allowedFileExtensions',$allowedFileExtensions) - ->setAttribute('enabled',$enabled) - ->setAttribute('encryption',$encryption) - ->setAttribute('antiVirus',$antiVirus) + ->setAttribute('name', $name) + ->setAttribute('$read', $read) + ->setAttribute('$write', $write) + ->setAttribute('maximumFileSize', $maximumFileSize) + ->setAttribute('allowedFileExtensions', $allowedFileExtensions) + ->setAttribute('enabled', $enabled) + ->setAttribute('encryption', $encryption) + ->setAttribute('antiVirus', $antiVirus) ); $audits @@ -229,7 +229,7 @@ App::delete('/v1/storage/buckets/:bucketId') ->setParam('document', $bucket) ; - if(!$dbForInternal->deleteDocument('buckets', $bucketId)) { + if (!$dbForInternal->deleteDocument('buckets', $bucketId)) { throw new Exception('Failed to remove project from DB', 500); } @@ -247,7 +247,7 @@ App::delete('/v1/storage/buckets/:bucketId') }); App::post('/v1/storage/buckets/:bucketId/files') - ->alias('/v1/storage/files',['bucketId' => 'default']) + ->alias('/v1/storage/files', ['bucketId' => 'default']) ->desc('Create File') ->groups(['api', 'storage']) ->label('scope', 'files.write') @@ -281,20 +281,20 @@ App::post('/v1/storage/buckets/:bucketId/files') $bucket = $dbForInternal->getDocument('buckets', $bucketId); - if($bucket->isEmpty()) { + if ($bucket->isEmpty()) { throw new Exception('Bucket not found', 404); } $file = $request->getFiles('file'); /* - * Validators - */ + * Validators + */ $allowedFileExtensions = $bucket->getAttribute('allowedFileExtensions', []); $fileExt = new FileExt($allowedFileExtensions); $maximumFileSize = $bucket->getAttribute('maximumFileSize', 0); - if($maximumFileSize > (int) App::getEnv('_APP_STORAGE_LIMIT',0)) { + if ($maximumFileSize > (int) App::getEnv('_APP_STORAGE_LIMIT', 0)) { throw new Exception('Error bucket maximum file size is larger than _APP_STORAGE_LIMIT', 500); } @@ -306,97 +306,198 @@ App::post('/v1/storage/buckets/:bucketId/files') } // Make sure we handle a single file and multiple files the same way - $file['name'] = (\is_array($file['name']) && isset($file['name'][0])) ? $file['name'][0] : $file['name']; - $file['tmp_name'] = (\is_array($file['tmp_name']) && isset($file['tmp_name'][0])) ? $file['tmp_name'][0] : $file['tmp_name']; - $file['size'] = (\is_array($file['size']) && isset($file['size'][0])) ? $file['size'][0] : $file['size']; + $fileName = (\is_array($file['name']) && isset($file['name'][0])) ? $file['name'][0] : $file['name']; + $fileTmpName = (\is_array($file['tmp_name']) && isset($file['tmp_name'][0])) ? $file['tmp_name'][0] : $file['tmp_name']; + $size = (\is_array($file['size']) && isset($file['size'][0])) ? $file['size'][0] : $file['size']; + + $contentRange = $request->getHeader('content-range'); + $uploadId = \uniqid(); + $chunk = 1; + $chunks = 1; + + if (!empty($contentRange)) { + $uploadId = empty($request->getHeader('x-appwrite-upload-id')) ? $uploadId : $request->getHeader('x-appwrite-upload-id'); + $contentRange = explode(" ", $contentRange); + if (count($contentRange) != 2) { + throw new Exception('Invalid content-range header', 400); + } + + $rangeData = explode("/", $contentRange[1]); + if (count($rangeData) != 2) { + throw new Exception('Invalid content-range header', 400); + } + + $size = (int) $rangeData[1]; + $parts = explode("-", $rangeData[0]); + if (count($parts) != 2) { + throw new Exception('Invalid content-range header', 400); + } + + $start = (int) $parts[0]; + $end = (int) $parts[1]; + if ($start > $end || $end > $size) { + throw new Exception('Invalid content-range header', 400); + } + + if ($end == $size) { + $chunks = $chunk = -1; + } else { + $chunks = (int) ceil($size / ($end + 1 - $start)); + $chunk = (int) ($start / ($end + 1 - $start)); + } + } // Check if file type is allowed (feature for project settings?) - if (!empty($allowedFileExtensions) && !$fileExt->isValid($file['name'])) { + if (!empty($allowedFileExtensions) && !$fileExt->isValid($fileName)) { throw new Exception('File extension not allowed', 400); } - if (!$fileSize->isValid($file['size'])) { // Check if file size is exceeding allowed limit + if (!$fileSize->isValid($size)) { // Check if file size is exceeding allowed limit throw new Exception('File size not allowed', 400); } $device = Storage::getDevice('files'); - if (!$upload->isValid($file['tmp_name'])) { + if (!$upload->isValid($fileTmpName)) { throw new Exception('Invalid file', 403); } // Save to storage - $size = $device->getFileSize($file['tmp_name']); - $path = $device->getPath(\uniqid().'.'.\pathinfo($file['name'], PATHINFO_EXTENSION)); - $path = $bucket->getId() . '/' . $path; - - if (!$device->upload($file['tmp_name'], $path)) { // TODO deprecate 'upload' and replace with 'move' - throw new Exception('Failed moving file', 500); - } + $size = $size ?? $device->getFileSize($fileTmpName); + $path = $device->getPath($uploadId . '.' . \pathinfo($fileName, PATHINFO_EXTENSION)); + $path = $bucket->getId() . $path; - $mimeType = $device->getFileMimeType($path); // Get mime-type before compression and encryption + $file = $dbForInternal->getDocument('files', $uploadId); - if (App::getEnv('_APP_STORAGE_ANTIVIRUS') === 'enabled' && $bucket->getAttribute('antiVirus', true) && $size <= APP_LIMIT_ANTIVIRUS) { - $antiVirus = new Network(App::getEnv('_APP_STORAGE_ANTIVIRUS_HOST', 'clamav'), - (int) App::getEnv('_APP_STORAGE_ANTIVIRUS_PORT', 3310)); - - if (!$antiVirus->fileScan($path)) { - $device->delete($path); - throw new Exception('Invalid file', 403); + if (!$file->isEmpty()) { + $chunks = $file->getAttribute('totalChunks', 1); + if ($chunk == -1) { + $chunk = $chunks - 1; } } - // Compression - $data = $device->read($path); - if($size <= APP_LIMIT_COMPRESSION) { - $compressor = new GZIP(); - $data = $compressor->compress($data); - } - - if($bucket->getAttribute('encryption', true) && $size <= APP_LIMIT_ENCRYPTION) { - $key = App::getEnv('_APP_OPENSSL_KEY_V1'); - $iv = OpenSSL::randomPseudoBytes(OpenSSL::cipherIVLength(OpenSSL::CIPHER_AES_128_GCM)); - $data = OpenSSL::encrypt($data, OpenSSL::CIPHER_AES_128_GCM, $key, 0, $iv, $tag); + $uploadedChunks = $device->upload($fileTmpName, $path, $chunk, $chunks); + if (empty($uploadedChunks)) { + throw new Exception('Failed uploading file', 500); } - if (!$device->write($path, $data, $mimeType)) { - throw new Exception('Failed to save file', 500); + if ($uploadedChunks == $chunks) { + $mimeType = $device->getFileMimeType($path); // Get mime-type before compression and encryption + + if (App::getEnv('_APP_STORAGE_ANTIVIRUS') === 'enabled' && $bucket->getAttribute('antiVirus', true) && $size <= APP_LIMIT_ANTIVIRUS) { + $antiVirus = new Network(App::getEnv('_APP_STORAGE_ANTIVIRUS_HOST', 'clamav'), + (int) App::getEnv('_APP_STORAGE_ANTIVIRUS_PORT', 3310)); + + if (!$antiVirus->fileScan($path)) { + $device->delete($path); + throw new Exception('Invalid file', 403); + } + } + + // Compression + $data = $device->read($path); + if ($size <= APP_LIMIT_COMPRESSION) { + $compressor = new GZIP(); + $data = $compressor->compress($data); + } + + if ($bucket->getAttribute('encryption', true) && $size <= APP_LIMIT_ENCRYPTION) { + $key = App::getEnv('_APP_OPENSSL_KEY_V1'); + $iv = OpenSSL::randomPseudoBytes(OpenSSL::cipherIVLength(OpenSSL::CIPHER_AES_128_GCM)); + $data = OpenSSL::encrypt($data, OpenSSL::CIPHER_AES_128_GCM, $key, 0, $iv, $tag); + } + + if (!$device->write($path, $data, $mimeType)) { + throw new Exception('Failed to save file', 500); + } + + $sizeActual = $device->getFileSize($path); + + $read = (is_null($read) && !$user->isEmpty()) ? ['user:' . $user->getId()] : $read ?? []; + $write = (is_null($write) && !$user->isEmpty()) ? ['user:' . $user->getId()] : $write ?? []; + $algorithm = empty($compressor) ? '' : $compressor->getName(); + $fileHash = $device->getFileHash($path); + + if ($bucket->getAttribute('encryption', true) && $size <= APP_LIMIT_ENCRYPTION) { + $openSSLVersion = '1'; + $openSSLCipher = OpenSSL::CIPHER_AES_128_GCM; + $openSSLTag = \bin2hex($tag); + $openSSLIV = \bin2hex($iv); + } + + if ($file->isEmpty()) { + $data = [ + '$read' => $read, + '$write' => $write, + 'dateCreated' => \time(), + 'bucketId' => $bucket->getId(), + 'name' => $fileName, + 'path' => $path, + 'signature' => $fileHash, + 'mimeType' => $mimeType, + 'sizeOriginal' => $size, + 'sizeActual' => $sizeActual, + 'algorithm' => $algorithm, + 'comment' => '', + 'totalChunks' => $chunks, + 'uploadedChunks' => $uploadedChunks, + 'openSSLVersion' => $openSSLVersion, + 'openSSLCipher' => $openSSLCipher, + 'openSSLTag' => $openSSLTag, + 'openSSLIV' => $openSSLIV, + ]; + $file = $dbForInternal->createDocument('files', new Document($data)); + } else { + $file = $dbForInternal->updateDocument('files', $uploadId, $file + ->setAttribute('$read', $read) + ->setAttribute('$write', $write) + ->setAttribute('signature', $fileHash) + ->setAttribute('mimeType', $mimeType) + ->setAttribute('sizeActual', $sizeActual) + ->setAttribute('algorithm', $algorithm) + ->setAttribute('openSSLVersion', $openSSLVersion) + ->setAttribute('openSSLCipher', $openSSLCipher) + ->setAttribute('openSSLTag', $openSSLTag) + ->setAttribute('openSSLIV', $openSSLIV) + ); + } + } else { + if ($file->isEmpty()) { + $data = [ + '$id' => $uploadId, + '$read' => (is_null($read) && !$user->isEmpty()) ? ['user:' . $user->getId()] : $read ?? [], // By default set read permissions for user + '$write' => (is_null($write) && !$user->isEmpty()) ? ['user:' . $user->getId()] : $write ?? [], // By default set write permissions for user + 'dateCreated' => \time(), + 'bucketId' => $bucket->getId(), + 'name' => $fileName, + 'path' => $path, + 'signature' => '', + 'mimeType' => '', + 'sizeOriginal' => $size, + 'sizeActual' => 0, + 'algorithm' => '', + 'comment' => '', + 'totalChunks' => $chunks, + 'uploadedChunks' => $uploadedChunks, + ]; + $file = $dbForInternal->createDocument('files', new Document($data)); + } else { + $file = $dbForInternal->updateDocument('files', $uploadId, $file + ->setAttribute('uploadedChunks', $uploadedChunks) + ); + } } - $sizeActual = $device->getFileSize($path); - - $data = [ - '$read' => (is_null($read) && !$user->isEmpty()) ? ['user:'.$user->getId()] : $read ?? [], // By default set read permissions for user - '$write' => (is_null($write) && !$user->isEmpty()) ? ['user:'.$user->getId()] : $write ?? [], // By default set write permissions for user - 'dateCreated' => \time(), - 'bucketId' => $bucket->getId(), - 'name' => $file['name'], - 'path' => $path, - 'signature' => $device->getFileHash($path), - 'mimeType' => $mimeType, - 'sizeOriginal' => $size, - 'sizeActual' => $sizeActual, - 'algorithm' => empty($compressor) ? '' : $compressor->getName(), - 'comment' => '', - ]; - - if($bucket->getAttribute('encryption', true) && $size <= APP_LIMIT_ENCRYPTION) { - $data['openSSLVersion'] = '1'; - $data['openSSLCipher'] = OpenSSL::CIPHER_AES_128_GCM; - $data['openSSLTag'] = \bin2hex($tag); - $data['openSSLIV'] = \bin2hex($iv); - } - - $file = $dbForInternal->createDocument('files', new Document($data)); - $audits ->setParam('event', 'storage.files.create') - ->setParam('resource', 'storage/files/'.$file->getId()) + ->setParam('resource', 'storage/files/' . $file->getId()) ; - $usage - ->setParam('storage', $sizeActual) - ; + if (!empty($sizeActual)) { + $usage + ->setParam('storage', $sizeActual) + ; + } $response->setStatusCode(Response::STATUS_CODE_CREATED); $response->dynamic2($file, Response::MODEL_FILE); @@ -427,13 +528,13 @@ App::get('/v1/storage/buckets/:bucketId/files') $bucket = $dbForInternal->getDocument('buckets', $bucketId); - if($bucket->isEmpty()) { + if ($bucket->isEmpty()) { throw new Exception('Bucket not found', 404); } $queries = [new Query('bucketId', Query::TYPE_EQUAL, [$bucketId])]; - if($search) { + if ($search) { $queries[] = [new Query('name', Query::TYPE_SEARCH, [$search])]; } @@ -465,13 +566,13 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId') $bucket = $dbForInternal->getDocument('buckets', $bucketId); - if($bucket->isEmpty()) { + if ($bucket->isEmpty()) { throw new Exception('Bucket not found', 404); } $file = $dbForInternal->getDocument('files', $fileId); - if ($file->isEmpty() || $file->getAttribute('bucketId') != $bucketId) { + if ($file->isEmpty() || $file->getAttribute('bucketId') != $bucketId) { throw new Exception('File not found', 404); } @@ -499,8 +600,8 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview') ->param('borderWidth', 0, new Range(0, 100), 'Preview image border in pixels. Pass an integer between 0 to 100. Defaults to 0.', true) ->param('borderColor', '', new HexColor(), 'Preview image border color. Use a valid HEX color, no # is needed for prefix.', true) ->param('borderRadius', 0, new Range(0, 4000), 'Preview image border radius in pixels. Pass an integer between 0 to 4000.', true) - ->param('opacity', 1, new Range(0,1, Range::TYPE_FLOAT), 'Preview image opacity. Only works with images having an alpha channel (like png). Pass a number between 0 to 1.', true) - ->param('rotation', 0, new Range(0,360), 'Preview image rotation in degrees. Pass an integer between 0 and 360.', true) + ->param('opacity', 1, new Range(0, 1, Range::TYPE_FLOAT), 'Preview image opacity. Only works with images having an alpha channel (like png). Pass a number between 0 to 1.', true) + ->param('rotation', 0, new Range(0, 360), 'Preview image rotation in degrees. Pass an integer between 0 and 360.', true) ->param('background', '', new HexColor(), 'Preview image background color. Only works with transparent images (png). Use a valid HEX color, no # is needed for prefix.', true) ->param('output', '', new WhiteList(\array_keys(Config::getParam('storage-outputs')), true), 'Output format type (jpeg, jpg, png, gif and webp).', true) ->inject('request') @@ -524,7 +625,7 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview') } $bucket = $dbForInternal->getDocument('buckets', $bucketId); - if($bucket->isEmpty()) { + if ($bucket->isEmpty()) { throw new Exception('Bucket not found', 404); } @@ -536,8 +637,8 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview') $outputs = Config::getParam('storage-outputs'); $fileLogos = Config::getParam('storage-logos'); - $date = \date('D, d M Y H:i:s', \time() + (60 * 60 * 24 * 45)).' GMT'; // 45 days cache - $key = \md5($fileId.$width.$height.$quality.$borderWidth.$borderColor.$borderRadius.$opacity.$rotation.$background.$storage.$output); + $date = \date('D, d M Y H:i:s', \time() + (60 * 60 * 24 * 45)) . ' GMT'; // 45 days cache + $key = \md5($fileId . $width . $height . $quality . $borderWidth . $borderColor . $borderRadius . $opacity . $rotation . $background . $storage . $output); $file = $dbForInternal->getDocument('files', $fileId); @@ -557,7 +658,7 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview') $cipher = null; $background = (empty($background)) ? 'eceff1' : $background; $type = \strtolower(\pathinfo($path, PATHINFO_EXTENSION)); - $key = \md5($path.$width.$height.$quality.$borderWidth.$borderColor.$borderRadius.$opacity.$rotation.$background.$storage.$output); + $key = \md5($path . $width . $height . $quality . $borderWidth . $borderColor . $borderRadius . $opacity . $rotation . $background . $storage . $output); } $compressor = new GZIP(); @@ -567,8 +668,8 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview') throw new Exception('File not found', 404); } - $cache = new Cache(new Filesystem(APP_STORAGE_CACHE.'/app-'.$project->getId())); // Limit file number or size - $data = $cache->load($key, 60 * 60 * 24 * 30 * 3 /* 3 months */); + $cache = new Cache(new Filesystem(APP_STORAGE_CACHE . '/app-' . $project->getId())); // Limit file number or size + $data = $cache->load($key, 60 * 60 * 24 * 30 * 3/* 3 months */); if ($data) { $output = (empty($output)) ? $type : $output; @@ -587,7 +688,7 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview') $source = OpenSSL::decrypt( $source, $file->getAttribute('openSSLCipher'), - App::getEnv('_APP_OPENSSL_KEY_V'.$file->getAttribute('openSSLVersion')), + App::getEnv('_APP_OPENSSL_KEY_V' . $file->getAttribute('openSSLVersion')), 0, \hex2bin($file->getAttribute('openSSLIV')), \hex2bin($file->getAttribute('openSSLTag')) @@ -601,17 +702,17 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview') $image = new Image($source); $image->crop((int) $width, (int) $height, $gravity); - - if (!empty($opacity) || $opacity==0) { + + if (!empty($opacity) || $opacity == 0) { $image->setOpacity($opacity); } if (!empty($background)) { - $image->setBackground('#'.$background); + $image->setBackground('#' . $background); } - - if (!empty($borderWidth) ) { - $image->setBorder($borderWidth, '#'.$borderColor); + + if (!empty($borderWidth)) { + $image->setBorder($borderWidth, '#' . $borderColor); } if (!empty($borderRadius)) { @@ -660,7 +761,7 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/download') $bucket = $dbForInternal->getDocument('buckets', $bucketId); - if($bucket->isEmpty()) { + if ($bucket->isEmpty()) { throw new Exception('Bucket not found', 404); } @@ -673,7 +774,7 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/download') $path = $file->getAttribute('path', ''); if (!\file_exists($path)) { - throw new Exception('File not found in '.$path, 404); + throw new Exception('File not found in ' . $path, 404); } $device = Storage::getDevice('files'); @@ -683,13 +784,13 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/download') $source = OpenSSL::decrypt( $source, $file->getAttribute('openSSLCipher'), - App::getEnv('_APP_OPENSSL_KEY_V'.$file->getAttribute('openSSLVersion')), + App::getEnv('_APP_OPENSSL_KEY_V' . $file->getAttribute('openSSLVersion')), 0, \hex2bin($file->getAttribute('openSSLIV')), \hex2bin($file->getAttribute('openSSLTag')) ); } - if(!empty($file->getAttribute('algorithm', ''))) { + if (!empty($file->getAttribute('algorithm', ''))) { $compressor = new GZIP(); $source = $compressor->decompress($source); } @@ -697,8 +798,8 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/download') // Response $response ->setContentType($file->getAttribute('mimeType')) - ->addHeader('Content-Disposition', 'attachment; filename="'.$file->getAttribute('name', '').'"') - ->addHeader('Expires', \date('D, d M Y H:i:s', \time() + (60 * 60 * 24 * 45)).' GMT') // 45 days cache + ->addHeader('Content-Disposition', 'attachment; filename="' . $file->getAttribute('name', '') . '"') + ->addHeader('Expires', \date('D, d M Y H:i:s', \time() + (60 * 60 * 24 * 45)) . ' GMT') // 45 days cache ->addHeader('X-Peak', \memory_get_peak_usage()) ->send($source) ; @@ -726,11 +827,11 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/view') $bucket = $dbForInternal->getDocument('buckets', $bucketId); - if($bucket->isEmpty()) { + if ($bucket->isEmpty()) { throw new Exception('Bucket not found', 404); } - $file = $dbForInternal->getDocument('files', $fileId); + $file = $dbForInternal->getDocument('files', $fileId); $mimes = Config::getParam('storage-mimes'); if ($file->isEmpty() || $file->getAttribute('bucketId') != $bucketId) { @@ -740,7 +841,7 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/view') $path = $file->getAttribute('path', ''); if (!\file_exists($path)) { - throw new Exception('File not found in '.$path, 404); + throw new Exception('File not found in ' . $path, 404); } $compressor = new GZIP(); @@ -758,7 +859,7 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/view') $source = OpenSSL::decrypt( $source, $file->getAttribute('openSSLCipher'), - App::getEnv('_APP_OPENSSL_KEY_V'.$file->getAttribute('openSSLVersion')), + App::getEnv('_APP_OPENSSL_KEY_V' . $file->getAttribute('openSSLVersion')), 0, \hex2bin($file->getAttribute('openSSLIV')), \hex2bin($file->getAttribute('openSSLTag')) @@ -773,8 +874,8 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/view') ->setContentType($contentType) ->addHeader('Content-Security-Policy', 'script-src none;') ->addHeader('X-Content-Type-Options', 'nosniff') - ->addHeader('Content-Disposition', 'inline; filename="'.$fileName.'"') - ->addHeader('Expires', \date('D, d M Y H:i:s', \time() + (60 * 60 * 24 * 45)).' GMT') // 45 days cache + ->addHeader('Content-Disposition', 'inline; filename="' . $fileName . '"') + ->addHeader('Expires', \date('D, d M Y H:i:s', \time() + (60 * 60 * 24 * 45)) . ' GMT') // 45 days cache ->addHeader('X-Peak', \memory_get_peak_usage()) ->send($output) ; @@ -807,7 +908,7 @@ App::put('/v1/storage/buckets/:bucketId/files/:fileId') $bucket = $dbForInternal->getDocument('buckets', $bucketId); - if($bucket->isEmpty()) { + if ($bucket->isEmpty()) { throw new Exception('Bucket not found', 404); } @@ -824,7 +925,7 @@ App::put('/v1/storage/buckets/:bucketId/files/:fileId') $audits ->setParam('event', 'storage.files.update') - ->setParam('resource', 'storage/files/'.$file->getId()) + ->setParam('resource', 'storage/files/' . $file->getId()) ; $response->dynamic2($file, Response::MODEL_FILE); @@ -855,10 +956,10 @@ App::delete('/v1/storage/buckets/:bucketId/files/:fileId') /** @var Appwrite\Event\Event $events */ /** @var Appwrite\Event\Event $audits */ /** @var Appwrite\Event\Event $usage */ - + $bucket = $dbForInternal->getDocument('buckets', $bucketId); - if($bucket->isEmpty()) { + if ($bucket->isEmpty()) { throw new Exception('Bucket not found', 404); } @@ -875,10 +976,10 @@ App::delete('/v1/storage/buckets/:bucketId/files/:fileId') throw new Exception('Failed to remove file from DB', 500); } } - + $audits ->setParam('event', 'storage.files.delete') - ->setParam('resource', 'storage/files/'.$file->getId()) + ->setParam('resource', 'storage/files/' . $file->getId()) ; $usage diff --git a/composer.json b/composer.json index f0652c4403..b932a3da9f 100644 --- a/composer.json +++ b/composer.json @@ -51,7 +51,7 @@ "utopia-php/preloader": "0.2.*", "utopia-php/domains": "1.1.*", "utopia-php/swoole": "0.2.*", - "utopia-php/storage": "0.5.*", + "utopia-php/storage": "dev-feat-large-file-support", "utopia-php/image": "0.5.*", "resque/php-resque": "1.3.6", "matomo/device-detector": "4.2.3", @@ -70,6 +70,10 @@ { "type": "git", "url": "https://github.com/lohanidamodar/audit" + }, + { + "type": "git", + "url": "https://github.com/utopia-php/storage" } ], "require-dev": { diff --git a/composer.lock b/composer.lock index 4553027855..3c2e595ba2 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "14172d7cbfa5a5b7353d3f8b24e44b81", + "content-hash": "8c63d8df231b87f77b8ee5b8e10242bc", "packages": [ { "name": "adhocore/jwt", @@ -2337,17 +2337,11 @@ }, { "name": "utopia-php/storage", - "version": "0.5.0", + "version": "dev-feat-large-file-support", "source": { "type": "git", - "url": "https://github.com/utopia-php/storage.git", - "reference": "92ae20c7a2ac329f573a58a82dc245134cc63408" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/utopia-php/storage/zipball/92ae20c7a2ac329f573a58a82dc245134cc63408", - "reference": "92ae20c7a2ac329f573a58a82dc245134cc63408", - "shasum": "" + "url": "https://github.com/utopia-php/storage", + "reference": "2e910d935c29cef80ee71b004bcd20dc2e219501" }, "require": { "php": ">=7.4", @@ -2363,7 +2357,6 @@ "Utopia\\Storage\\": "src/Storage" } }, - "notification-url": "https://packagist.org/downloads/", "license": [ "MIT" ], @@ -2381,11 +2374,7 @@ "upf", "utopia" ], - "support": { - "issues": "https://github.com/utopia-php/storage/issues", - "source": "https://github.com/utopia-php/storage/tree/0.5.0" - }, - "time": "2021-04-15T16:43:12+00:00" + "time": "2021-07-07T08:29:23+00:00" }, { "name": "utopia-php/swoole", @@ -6235,7 +6224,8 @@ "minimum-stability": "stable", "stability-flags": { "utopia-php/abuse": 20, - "utopia-php/audit": 20 + "utopia-php/audit": 20, + "utopia-php/storage": 20 }, "prefer-stable": false, "prefer-lowest": false, diff --git a/src/Appwrite/Utopia/Response/Model/File.php b/src/Appwrite/Utopia/Response/Model/File.php index 302bb8a61e..50b8f28a99 100644 --- a/src/Appwrite/Utopia/Response/Model/File.php +++ b/src/Appwrite/Utopia/Response/Model/File.php @@ -60,6 +60,18 @@ class File extends Model 'default' => 0, 'example' => 17890, ]) + ->addRule('totalChunks', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Total number of chunks available', + 'default' => 0, + 'example' => 17890, + ]) + ->addRule('uploadedChunks', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Total number of chunks uploaded', + 'default' => 0, + 'example' => 17890, + ]) ; } diff --git a/tests/e2e/Services/Storage/StorageBase.php b/tests/e2e/Services/Storage/StorageBase.php index 9f35e0fe0f..7a6629f124 100644 --- a/tests/e2e/Services/Storage/StorageBase.php +++ b/tests/e2e/Services/Storage/StorageBase.php @@ -77,6 +77,49 @@ trait StorageBase $this->assertEquals('video/mp4', $file2['body']['mimeType']); $this->assertEquals(23660615, $file2['body']['sizeOriginal']); $this->assertEquals(md5_file(realpath(__DIR__ . '/../../../resources/disk-a/large-file.mp4')), $file2['body']['signature']); // should validate that the file is not encrypted + + /** + * Chunked Upload + */ + + $source = __DIR__ . "/../../../resources/disk-a/large-file.mp4"; + $totalSize = \filesize($source); + $chunkSize = 5000000; + $start = 0; + $handle = @fopen($source, "rb"); + $uploadId = ''; + $op = __DIR__ . '/chunk.part'; + while ($start < $totalSize) { + $contents = fread($handle, $chunkSize); + $cc = fopen($op, 'wb'); + fwrite($cc, $contents); + fclose($cc); + $curlFile = new CURLFile($op, 'video/mp4', 'large-file.mp4'); + $contentRanges = 'bytes ' . $start . '-' . min((($start + $chunkSize) - 1), $totalSize) . '/' . $totalSize; + $largeFile = $this->client->call(Client::METHOD_POST, '/storage/buckets/' . $bucket2['body']['$id'] . '/files', array_merge([ + 'content-type' => 'multipart/form-data', + 'x-appwrite-project' => $this->getProject()['$id'], + 'content-range' => $contentRanges, + 'x-appwrite-upload-id' => $uploadId, + ], $this->getHeaders()), [ + 'file' => $curlFile, + 'read' => ['role:all'], + 'write' => ['role:all'], + ]); + $uploadId = $largeFile['body']['$id']; + $start += strlen($contents); + fseek($handle, $start); + } + \unlink($op); + @fclose($handle); + + $this->assertEquals(201, $largeFile['headers']['status-code']); + $this->assertNotEmpty($largeFile['body']['$id']); + $this->assertIsInt($largeFile['body']['dateCreated']); + $this->assertEquals('large-file.mp4', $largeFile['body']['name']); + $this->assertEquals('video/mp4', $largeFile['body']['mimeType']); + $this->assertEquals($totalSize, $largeFile['body']['sizeOriginal']); + $this->assertEquals(md5_file(realpath(__DIR__ . '/../../../resources/disk-a/large-file.mp4')), $largeFile['body']['signature']); // should validate that the file is not encrypted /** * Test for FAILURE unknown Bucket