CodeMirror/appwrite
1
0
Fork 0
mirror of synced 2024-07-02 13:10:38 +12:00
Code Issues Projects Releases Wiki Activity

first working POC of large file upload

Browse source
This commit is contained in:
Damodar Lohani 2021-07-07 15:52:11 +05:45
parent 8d40a1b3ce
commit 5db04e09f9
6 changed files with 324 additions and 154 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
app/config/collections2.php
View file

@ -625,6 +625,26 @@ $collections = [
'array' => false,
'filters' => [],
],
[
'$id' => 'totalChunks',
'type' => Database::VAR_INTEGER,
'format' => '',
'size' => 0,
'signed' => false,
'required' => false,
'array' => false,
'filters' => [],
],
[
'$id' => 'uploadedChunks',
'type' => Database::VAR_INTEGER,
'format' => '',
'size' => 0,
'signed' => false,
'required' => false,
'array' => false,
'filters' => [],
],
],
'indexes' => [
[

373
app/controllers/api/storage.php
View file

@ -1,30 +1,30 @@
<?php
use Utopia\App;
use Utopia\Exception;
use Utopia\Validator\ArrayList;
use Utopia\Validator\WhiteList;
use Utopia\Validator\Range;
use Utopia\Validator\Text;
use Utopia\Validator\Boolean;
use Utopia\Validator\HexColor;
use Utopia\Cache\Cache;
use Utopia\Cache\Adapter\Filesystem;
use Appwrite\ClamAV\Network;
use Utopia\Database\Document;
use Appwrite\Database\Validator\UID;
use Utopia\Storage\Storage;
use Utopia\Storage\Validator\File;
use Utopia\Storage\Validator\FileSize;
use Utopia\Storage\Validator\Upload;
use Utopia\Storage\Compression\Algorithms\GZIP;
use Utopia\Image\Image;
use Appwrite\OpenSSL\OpenSSL;
use Appwrite\Utopia\Response;
use Utopia\App;
use Utopia\Cache\Adapter\Filesystem;
use Utopia\Cache\Cache;
use Utopia\Config\Config;
use Utopia\Validator\Integer;
use Utopia\Database\Document;
use Utopia\Database\Query;
use Utopia\Exception;
use Utopia\Image\Image;
use Utopia\Storage\Compression\Algorithms\GZIP;
use Utopia\Storage\Storage;
use Utopia\Storage\Validator\File;
use Utopia\Storage\Validator\FileExt;
use Utopia\Storage\Validator\FileSize;
use Utopia\Storage\Validator\Upload;
use Utopia\Validator\ArrayList;
use Utopia\Validator\Boolean;
use Utopia\Validator\HexColor;
use Utopia\Validator\Integer;
use Utopia\Validator\Range;
use Utopia\Validator\Text;
use Utopia\Validator\WhiteList;
App::post('/v1/storage/buckets')
->desc('Create storage bucket')
@ -41,7 +41,7 @@ App::post('/v1/storage/buckets')
->param('name', '', new Text(128), 'Bucket name', false)
->param('read', [], new ArrayList(new Text(64)), 'An array of strings with read permissions. By default no user is granted with any read permissions. [learn more about permissions](/docs/permissions) and get a full list of available permissions.', true)
->param('write', [], new ArrayList(new Text(64)), 'An array of strings with write permissions. By default no user is granted with any write permissions. [learn more about permissions](/docs/permissions) and get a full list of available permissions.', true)
->param('maximumFileSize', (int) App::getEnv('_APP_STORAGE_LIMIT', 0) , new Integer(), 'Maximum file size allowed in bytes. Maximum allowed value is ' . App::getEnv('_APP_STORAGE_LIMIT', 0) . '. For self-hosted setups you can change the max limit by changing the `_APP_STORAGE_LIMIT` environment variable. [Learn more about storage environment variables](docs/environment-variables#storage)', true)
->param('maximumFileSize', (int) App::getEnv('_APP_STORAGE_LIMIT', 0), new Integer(), 'Maximum file size allowed in bytes. Maximum allowed value is ' . App::getEnv('_APP_STORAGE_LIMIT', 0) . '. For self-hosted setups you can change the max limit by changing the `_APP_STORAGE_LIMIT` environment variable. [Learn more about storage environment variables](docs/environment-variables#storage)', true)
->param('allowedFileExtensions', [], new ArrayList(new Text(64)), 'Allowed file extensions', true)
->param('enabled', true, new Boolean(), 'Is bucket enabled?', true)
->param('adapter', 'local', new WhiteList(['local']), 'Storage adapter.', true)
@ -171,18 +171,18 @@ App::put('/v1/storage/buckets/:bucketId')
throw new Exception('Bucket not found', 404);
}
$read ??= $bucket->getAttribute('$read', []); // By default inherit read permissions
$write ??= $bucket->getAttribute('$write',[]); // By default inherit write permissions
$read??=$bucket->getAttribute('$read', []); // By default inherit read permissions
$write??=$bucket->getAttribute('$write', []); // By default inherit write permissions
$bucket = $dbForInternal->updateDocument('buckets', $bucket->getId(), $bucket
->setAttribute('name',$name)
->setAttribute('$read',$read)
->setAttribute('$write',$write)
->setAttribute('maximumFileSize',$maximumFileSize)
->setAttribute('allowedFileExtensions',$allowedFileExtensions)
->setAttribute('enabled',$enabled)
->setAttribute('encryption',$encryption)
->setAttribute('antiVirus',$antiVirus)
->setAttribute('name', $name)
->setAttribute('$read', $read)
->setAttribute('$write', $write)
->setAttribute('maximumFileSize', $maximumFileSize)
->setAttribute('allowedFileExtensions', $allowedFileExtensions)
->setAttribute('enabled', $enabled)
->setAttribute('encryption', $encryption)
->setAttribute('antiVirus', $antiVirus)
);
$audits
@ -229,7 +229,7 @@ App::delete('/v1/storage/buckets/:bucketId')
->setParam('document', $bucket)
;
if(!$dbForInternal->deleteDocument('buckets', $bucketId)) {
if (!$dbForInternal->deleteDocument('buckets', $bucketId)) {
throw new Exception('Failed to remove project from DB', 500);
}
@ -247,7 +247,7 @@ App::delete('/v1/storage/buckets/:bucketId')
});
App::post('/v1/storage/buckets/:bucketId/files')
->alias('/v1/storage/files',['bucketId' => 'default'])
->alias('/v1/storage/files', ['bucketId' => 'default'])
->desc('Create File')
->groups(['api', 'storage'])
->label('scope', 'files.write')
@ -281,20 +281,20 @@ App::post('/v1/storage/buckets/:bucketId/files')
$bucket = $dbForInternal->getDocument('buckets', $bucketId);
if($bucket->isEmpty()) {
if ($bucket->isEmpty()) {
throw new Exception('Bucket not found', 404);
}
$file = $request->getFiles('file');
/*
* Validators
*/
* Validators
*/
$allowedFileExtensions = $bucket->getAttribute('allowedFileExtensions', []);
$fileExt = new FileExt($allowedFileExtensions);
$maximumFileSize = $bucket->getAttribute('maximumFileSize', 0);
if($maximumFileSize > (int) App::getEnv('_APP_STORAGE_LIMIT',0)) {
if ($maximumFileSize > (int) App::getEnv('_APP_STORAGE_LIMIT', 0)) {
throw new Exception('Error bucket maximum file size is larger than _APP_STORAGE_LIMIT', 500);
}
@ -306,97 +306,198 @@ App::post('/v1/storage/buckets/:bucketId/files')
}
// Make sure we handle a single file and multiple files the same way
$file['name'] = (\is_array($file['name']) && isset($file['name'][0])) ? $file['name'][0] : $file['name'];
$file['tmp_name'] = (\is_array($file['tmp_name']) && isset($file['tmp_name'][0])) ? $file['tmp_name'][0] : $file['tmp_name'];
$file['size'] = (\is_array($file['size']) && isset($file['size'][0])) ? $file['size'][0] : $file['size'];
$fileName = (\is_array($file['name']) && isset($file['name'][0])) ? $file['name'][0] : $file['name'];
$fileTmpName = (\is_array($file['tmp_name']) && isset($file['tmp_name'][0])) ? $file['tmp_name'][0] : $file['tmp_name'];
$size = (\is_array($file['size']) && isset($file['size'][0])) ? $file['size'][0] : $file['size'];
$contentRange = $request->getHeader('content-range');
$uploadId = \uniqid();
$chunk = 1;
$chunks = 1;
if (!empty($contentRange)) {
$uploadId = empty($request->getHeader('x-appwrite-upload-id')) ? $uploadId : $request->getHeader('x-appwrite-upload-id');
$contentRange = explode(" ", $contentRange);
if (count($contentRange) != 2) {
throw new Exception('Invalid content-range header', 400);
}
$rangeData = explode("/", $contentRange[1]);
if (count($rangeData) != 2) {
throw new Exception('Invalid content-range header', 400);
}
$size = (int) $rangeData[1];
$parts = explode("-", $rangeData[0]);
if (count($parts) != 2) {
throw new Exception('Invalid content-range header', 400);
}
$start = (int) $parts[0];
$end = (int) $parts[1];
if ($start > $end || $end > $size) {
throw new Exception('Invalid content-range header', 400);
}
if ($end == $size) {
$chunks = $chunk = -1;
} else {
$chunks = (int) ceil($size / ($end + 1 - $start));
$chunk = (int) ($start / ($end + 1 - $start));
}
}
// Check if file type is allowed (feature for project settings?)
if (!empty($allowedFileExtensions) && !$fileExt->isValid($file['name'])) {
if (!empty($allowedFileExtensions) && !$fileExt->isValid($fileName)) {
throw new Exception('File extension not allowed', 400);
}
if (!$fileSize->isValid($file['size'])) { // Check if file size is exceeding allowed limit
if (!$fileSize->isValid($size)) { // Check if file size is exceeding allowed limit
throw new Exception('File size not allowed', 400);
}
$device = Storage::getDevice('files');
if (!$upload->isValid($file['tmp_name'])) {
if (!$upload->isValid($fileTmpName)) {
throw new Exception('Invalid file', 403);
}
// Save to storage
$size = $device->getFileSize($file['tmp_name']);
$path = $device->getPath(\uniqid().'.'.\pathinfo($file['name'], PATHINFO_EXTENSION));
$path = $bucket->getId() . '/' . $path;
if (!$device->upload($file['tmp_name'], $path)) { // TODO deprecate 'upload' and replace with 'move'
throw new Exception('Failed moving file', 500);
}
$size = $size ?? $device->getFileSize($fileTmpName);
$path = $device->getPath($uploadId . '.' . \pathinfo($fileName, PATHINFO_EXTENSION));
$path = $bucket->getId() . $path;
$mimeType = $device->getFileMimeType($path); // Get mime-type before compression and encryption
$file = $dbForInternal->getDocument('files', $uploadId);
if (App::getEnv('_APP_STORAGE_ANTIVIRUS') === 'enabled' && $bucket->getAttribute('antiVirus', true) && $size <= APP_LIMIT_ANTIVIRUS) {
$antiVirus = new Network(App::getEnv('_APP_STORAGE_ANTIVIRUS_HOST', 'clamav'),
(int) App::getEnv('_APP_STORAGE_ANTIVIRUS_PORT', 3310));
if (!$antiVirus->fileScan($path)) {
$device->delete($path);
throw new Exception('Invalid file', 403);
if (!$file->isEmpty()) {
$chunks = $file->getAttribute('totalChunks', 1);
if ($chunk == -1) {
$chunk = $chunks - 1;
}
}
// Compression
$data = $device->read($path);
if($size <= APP_LIMIT_COMPRESSION) {
$compressor = new GZIP();
$data = $compressor->compress($data);
}
if($bucket->getAttribute('encryption', true) && $size <= APP_LIMIT_ENCRYPTION) {
$key = App::getEnv('_APP_OPENSSL_KEY_V1');
$iv = OpenSSL::randomPseudoBytes(OpenSSL::cipherIVLength(OpenSSL::CIPHER_AES_128_GCM));
$data = OpenSSL::encrypt($data, OpenSSL::CIPHER_AES_128_GCM, $key, 0, $iv, $tag);
$uploadedChunks = $device->upload($fileTmpName, $path, $chunk, $chunks);
if (empty($uploadedChunks)) {
throw new Exception('Failed uploading file', 500);
}
if (!$device->write($path, $data, $mimeType)) {
throw new Exception('Failed to save file', 500);
if ($uploadedChunks == $chunks) {
$mimeType = $device->getFileMimeType($path); // Get mime-type before compression and encryption
if (App::getEnv('_APP_STORAGE_ANTIVIRUS') === 'enabled' && $bucket->getAttribute('antiVirus', true) && $size <= APP_LIMIT_ANTIVIRUS) {
$antiVirus = new Network(App::getEnv('_APP_STORAGE_ANTIVIRUS_HOST', 'clamav'),
(int) App::getEnv('_APP_STORAGE_ANTIVIRUS_PORT', 3310));
if (!$antiVirus->fileScan($path)) {
$device->delete($path);
throw new Exception('Invalid file', 403);
}
}
// Compression
$data = $device->read($path);
if ($size <= APP_LIMIT_COMPRESSION) {
$compressor = new GZIP();
$data = $compressor->compress($data);
}
if ($bucket->getAttribute('encryption', true) && $size <= APP_LIMIT_ENCRYPTION) {
$key = App::getEnv('_APP_OPENSSL_KEY_V1');
$iv = OpenSSL::randomPseudoBytes(OpenSSL::cipherIVLength(OpenSSL::CIPHER_AES_128_GCM));
$data = OpenSSL::encrypt($data, OpenSSL::CIPHER_AES_128_GCM, $key, 0, $iv, $tag);
}
if (!$device->write($path, $data, $mimeType)) {
throw new Exception('Failed to save file', 500);
}
$sizeActual = $device->getFileSize($path);
$read = (is_null($read) && !$user->isEmpty()) ? ['user:' . $user->getId()] : $read ?? [];
$write = (is_null($write) && !$user->isEmpty()) ? ['user:' . $user->getId()] : $write ?? [];
$algorithm = empty($compressor) ? '' : $compressor->getName();
$fileHash = $device->getFileHash($path);
if ($bucket->getAttribute('encryption', true) && $size <= APP_LIMIT_ENCRYPTION) {
$openSSLVersion = '1';
$openSSLCipher = OpenSSL::CIPHER_AES_128_GCM;
$openSSLTag = \bin2hex($tag);
$openSSLIV = \bin2hex($iv);
}
if ($file->isEmpty()) {
$data = [
'$read' => $read,
'$write' => $write,
'dateCreated' => \time(),
'bucketId' => $bucket->getId(),
'name' => $fileName,
'path' => $path,
'signature' => $fileHash,
'mimeType' => $mimeType,
'sizeOriginal' => $size,
'sizeActual' => $sizeActual,
'algorithm' => $algorithm,
'comment' => '',
'totalChunks' => $chunks,
'uploadedChunks' => $uploadedChunks,
'openSSLVersion' => $openSSLVersion,
'openSSLCipher' => $openSSLCipher,
'openSSLTag' => $openSSLTag,
'openSSLIV' => $openSSLIV,
];
$file = $dbForInternal->createDocument('files', new Document($data));
} else {
$file = $dbForInternal->updateDocument('files', $uploadId, $file
->setAttribute('$read', $read)
->setAttribute('$write', $write)
->setAttribute('signature', $fileHash)
->setAttribute('mimeType', $mimeType)
->setAttribute('sizeActual', $sizeActual)
->setAttribute('algorithm', $algorithm)
->setAttribute('openSSLVersion', $openSSLVersion)
->setAttribute('openSSLCipher', $openSSLCipher)
->setAttribute('openSSLTag', $openSSLTag)
->setAttribute('openSSLIV', $openSSLIV)
);
}
} else {
if ($file->isEmpty()) {
$data = [
'$id' => $uploadId,
'$read' => (is_null($read) && !$user->isEmpty()) ? ['user:' . $user->getId()] : $read ?? [], // By default set read permissions for user
'$write' => (is_null($write) && !$user->isEmpty()) ? ['user:' . $user->getId()] : $write ?? [], // By default set write permissions for user
'dateCreated' => \time(),
'bucketId' => $bucket->getId(),
'name' => $fileName,
'path' => $path,
'signature' => '',
'mimeType' => '',
'sizeOriginal' => $size,
'sizeActual' => 0,
'algorithm' => '',
'comment' => '',
'totalChunks' => $chunks,
'uploadedChunks' => $uploadedChunks,
];
$file = $dbForInternal->createDocument('files', new Document($data));
} else {
$file = $dbForInternal->updateDocument('files', $uploadId, $file
->setAttribute('uploadedChunks', $uploadedChunks)
);
}
}
$sizeActual = $device->getFileSize($path);
$data = [
'$read' => (is_null($read) && !$user->isEmpty()) ? ['user:'.$user->getId()] : $read ?? [], // By default set read permissions for user
'$write' => (is_null($write) && !$user->isEmpty()) ? ['user:'.$user->getId()] : $write ?? [], // By default set write permissions for user
'dateCreated' => \time(),
'bucketId' => $bucket->getId(),
'name' => $file['name'],
'path' => $path,
'signature' => $device->getFileHash($path),
'mimeType' => $mimeType,
'sizeOriginal' => $size,
'sizeActual' => $sizeActual,
'algorithm' => empty($compressor) ? '' : $compressor->getName(),
'comment' => '',
];
if($bucket->getAttribute('encryption', true) && $size <= APP_LIMIT_ENCRYPTION) {
$data['openSSLVersion'] = '1';
$data['openSSLCipher'] = OpenSSL::CIPHER_AES_128_GCM;
$data['openSSLTag'] = \bin2hex($tag);
$data['openSSLIV'] = \bin2hex($iv);
}
$file = $dbForInternal->createDocument('files', new Document($data));
$audits
->setParam('event', 'storage.files.create')
->setParam('resource', 'storage/files/'.$file->getId())
->setParam('resource', 'storage/files/' . $file->getId())
;
$usage
->setParam('storage', $sizeActual)
;
if (!empty($sizeActual)) {
$usage
->setParam('storage', $sizeActual)
;
}
$response->setStatusCode(Response::STATUS_CODE_CREATED);
$response->dynamic2($file, Response::MODEL_FILE);
@ -427,13 +528,13 @@ App::get('/v1/storage/buckets/:bucketId/files')
$bucket = $dbForInternal->getDocument('buckets', $bucketId);
if($bucket->isEmpty()) {
if ($bucket->isEmpty()) {
throw new Exception('Bucket not found', 404);
}
$queries = [new Query('bucketId', Query::TYPE_EQUAL, [$bucketId])];
if($search) {
if ($search) {
$queries[] = [new Query('name', Query::TYPE_SEARCH, [$search])];
}
@ -465,13 +566,13 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId')
$bucket = $dbForInternal->getDocument('buckets', $bucketId);
if($bucket->isEmpty()) {
if ($bucket->isEmpty()) {
throw new Exception('Bucket not found', 404);
}
$file = $dbForInternal->getDocument('files', $fileId);
if ($file->isEmpty() || $file->getAttribute('bucketId') != $bucketId) {
if ($file->isEmpty() || $file->getAttribute('bucketId') != $bucketId) {
throw new Exception('File not found', 404);
}
@ -499,8 +600,8 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview')
->param('borderWidth', 0, new Range(0, 100), 'Preview image border in pixels. Pass an integer between 0 to 100. Defaults to 0.', true)
->param('borderColor', '', new HexColor(), 'Preview image border color. Use a valid HEX color, no # is needed for prefix.', true)
->param('borderRadius', 0, new Range(0, 4000), 'Preview image border radius in pixels. Pass an integer between 0 to 4000.', true)
->param('opacity', 1, new Range(0,1, Range::TYPE_FLOAT), 'Preview image opacity. Only works with images having an alpha channel (like png). Pass a number between 0 to 1.', true)
->param('rotation', 0, new Range(0,360), 'Preview image rotation in degrees. Pass an integer between 0 and 360.', true)
->param('opacity', 1, new Range(0, 1, Range::TYPE_FLOAT), 'Preview image opacity. Only works with images having an alpha channel (like png). Pass a number between 0 to 1.', true)
->param('rotation', 0, new Range(0, 360), 'Preview image rotation in degrees. Pass an integer between 0 and 360.', true)
->param('background', '', new HexColor(), 'Preview image background color. Only works with transparent images (png). Use a valid HEX color, no # is needed for prefix.', true)
->param('output', '', new WhiteList(\array_keys(Config::getParam('storage-outputs')), true), 'Output format type (jpeg, jpg, png, gif and webp).', true)
->inject('request')
@ -524,7 +625,7 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview')
}
$bucket = $dbForInternal->getDocument('buckets', $bucketId);
if($bucket->isEmpty()) {
if ($bucket->isEmpty()) {
throw new Exception('Bucket not found', 404);
}
@ -536,8 +637,8 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview')
$outputs = Config::getParam('storage-outputs');
$fileLogos = Config::getParam('storage-logos');
$date = \date('D, d M Y H:i:s', \time() + (60 * 60 * 24 * 45)).' GMT'; // 45 days cache
$key = \md5($fileId.$width.$height.$quality.$borderWidth.$borderColor.$borderRadius.$opacity.$rotation.$background.$storage.$output);
$date = \date('D, d M Y H:i:s', \time() + (60 * 60 * 24 * 45)) . ' GMT'; // 45 days cache
$key = \md5($fileId . $width . $height . $quality . $borderWidth . $borderColor . $borderRadius . $opacity . $rotation . $background . $storage . $output);
$file = $dbForInternal->getDocument('files', $fileId);
@ -557,7 +658,7 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview')
$cipher = null;
$background = (empty($background)) ? 'eceff1' : $background;
$type = \strtolower(\pathinfo($path, PATHINFO_EXTENSION));
$key = \md5($path.$width.$height.$quality.$borderWidth.$borderColor.$borderRadius.$opacity.$rotation.$background.$storage.$output);
$key = \md5($path . $width . $height . $quality . $borderWidth . $borderColor . $borderRadius . $opacity . $rotation . $background . $storage . $output);
}
$compressor = new GZIP();
@ -567,8 +668,8 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview')
throw new Exception('File not found', 404);
}
$cache = new Cache(new Filesystem(APP_STORAGE_CACHE.'/app-'.$project->getId())); // Limit file number or size
$data = $cache->load($key, 60 * 60 * 24 * 30 * 3 /* 3 months */);
$cache = new Cache(new Filesystem(APP_STORAGE_CACHE . '/app-' . $project->getId())); // Limit file number or size
$data = $cache->load($key, 60 * 60 * 24 * 30 * 3/* 3 months */);
if ($data) {
$output = (empty($output)) ? $type : $output;
@ -587,7 +688,7 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview')
$source = OpenSSL::decrypt(
$source,
$file->getAttribute('openSSLCipher'),
App::getEnv('_APP_OPENSSL_KEY_V'.$file->getAttribute('openSSLVersion')),
App::getEnv('_APP_OPENSSL_KEY_V' . $file->getAttribute('openSSLVersion')),
0,
\hex2bin($file->getAttribute('openSSLIV')),
\hex2bin($file->getAttribute('openSSLTag'))
@ -601,17 +702,17 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview')
$image = new Image($source);
$image->crop((int) $width, (int) $height, $gravity);
if (!empty($opacity) || $opacity==0) {
if (!empty($opacity) || $opacity == 0) {
$image->setOpacity($opacity);
}
if (!empty($background)) {
$image->setBackground('#'.$background);
$image->setBackground('#' . $background);
}
if (!empty($borderWidth) ) {
$image->setBorder($borderWidth, '#'.$borderColor);
if (!empty($borderWidth)) {
$image->setBorder($borderWidth, '#' . $borderColor);
}
if (!empty($borderRadius)) {
@ -660,7 +761,7 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/download')
$bucket = $dbForInternal->getDocument('buckets', $bucketId);
if($bucket->isEmpty()) {
if ($bucket->isEmpty()) {
throw new Exception('Bucket not found', 404);
}
@ -673,7 +774,7 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/download')
$path = $file->getAttribute('path', '');
if (!\file_exists($path)) {
throw new Exception('File not found in '.$path, 404);
throw new Exception('File not found in ' . $path, 404);
}
$device = Storage::getDevice('files');
@ -683,13 +784,13 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/download')
$source = OpenSSL::decrypt(
$source,
$file->getAttribute('openSSLCipher'),
App::getEnv('_APP_OPENSSL_KEY_V'.$file->getAttribute('openSSLVersion')),
App::getEnv('_APP_OPENSSL_KEY_V' . $file->getAttribute('openSSLVersion')),
0,
\hex2bin($file->getAttribute('openSSLIV')),
\hex2bin($file->getAttribute('openSSLTag'))
);
}
if(!empty($file->getAttribute('algorithm', ''))) {
if (!empty($file->getAttribute('algorithm', ''))) {
$compressor = new GZIP();
$source = $compressor->decompress($source);
}
@ -697,8 +798,8 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/download')
// Response
$response
->setContentType($file->getAttribute('mimeType'))
->addHeader('Content-Disposition', 'attachment; filename="'.$file->getAttribute('name', '').'"')
->addHeader('Expires', \date('D, d M Y H:i:s', \time() + (60 * 60 * 24 * 45)).' GMT') // 45 days cache
->addHeader('Content-Disposition', 'attachment; filename="' . $file->getAttribute('name', '') . '"')
->addHeader('Expires', \date('D, d M Y H:i:s', \time() + (60 * 60 * 24 * 45)) . ' GMT') // 45 days cache
->addHeader('X-Peak', \memory_get_peak_usage())
->send($source)
;
@ -726,11 +827,11 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/view')
$bucket = $dbForInternal->getDocument('buckets', $bucketId);
if($bucket->isEmpty()) {
if ($bucket->isEmpty()) {
throw new Exception('Bucket not found', 404);
}
$file = $dbForInternal->getDocument('files', $fileId);
$file = $dbForInternal->getDocument('files', $fileId);
$mimes = Config::getParam('storage-mimes');
if ($file->isEmpty() || $file->getAttribute('bucketId') != $bucketId) {
@ -740,7 +841,7 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/view')
$path = $file->getAttribute('path', '');
if (!\file_exists($path)) {
throw new Exception('File not found in '.$path, 404);
throw new Exception('File not found in ' . $path, 404);
}
$compressor = new GZIP();
@ -758,7 +859,7 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/view')
$source = OpenSSL::decrypt(
$source,
$file->getAttribute('openSSLCipher'),
App::getEnv('_APP_OPENSSL_KEY_V'.$file->getAttribute('openSSLVersion')),
App::getEnv('_APP_OPENSSL_KEY_V' . $file->getAttribute('openSSLVersion')),
0,
\hex2bin($file->getAttribute('openSSLIV')),
\hex2bin($file->getAttribute('openSSLTag'))
@ -773,8 +874,8 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/view')
->setContentType($contentType)
->addHeader('Content-Security-Policy', 'script-src none;')
->addHeader('X-Content-Type-Options', 'nosniff')
->addHeader('Content-Disposition', 'inline; filename="'.$fileName.'"')
->addHeader('Expires', \date('D, d M Y H:i:s', \time() + (60 * 60 * 24 * 45)).' GMT') // 45 days cache
->addHeader('Content-Disposition', 'inline; filename="' . $fileName . '"')
->addHeader('Expires', \date('D, d M Y H:i:s', \time() + (60 * 60 * 24 * 45)) . ' GMT') // 45 days cache
->addHeader('X-Peak', \memory_get_peak_usage())
->send($output)
;
@ -807,7 +908,7 @@ App::put('/v1/storage/buckets/:bucketId/files/:fileId')
$bucket = $dbForInternal->getDocument('buckets', $bucketId);
if($bucket->isEmpty()) {
if ($bucket->isEmpty()) {
throw new Exception('Bucket not found', 404);
}
@ -824,7 +925,7 @@ App::put('/v1/storage/buckets/:bucketId/files/:fileId')
$audits
->setParam('event', 'storage.files.update')
->setParam('resource', 'storage/files/'.$file->getId())
->setParam('resource', 'storage/files/' . $file->getId())
;
$response->dynamic2($file, Response::MODEL_FILE);
@ -855,10 +956,10 @@ App::delete('/v1/storage/buckets/:bucketId/files/:fileId')
/** @var Appwrite\Event\Event $events */
/** @var Appwrite\Event\Event $audits */
/** @var Appwrite\Event\Event $usage */
$bucket = $dbForInternal->getDocument('buckets', $bucketId);
if($bucket->isEmpty()) {
if ($bucket->isEmpty()) {
throw new Exception('Bucket not found', 404);
}
@ -875,10 +976,10 @@ App::delete('/v1/storage/buckets/:bucketId/files/:fileId')
throw new Exception('Failed to remove file from DB', 500);
}
}
$audits
->setParam('event', 'storage.files.delete')
->setParam('resource', 'storage/files/'.$file->getId())
->setParam('resource', 'storage/files/' . $file->getId())
;
$usage

6
composer.json
View file

@ -51,7 +51,7 @@
"utopia-php/preloader": "0.2.*",
"utopia-php/domains": "1.1.*",
"utopia-php/swoole": "0.2.*",
"utopia-php/storage": "0.5.*",
"utopia-php/storage": "dev-feat-large-file-support",
"utopia-php/image": "0.5.*",
"resque/php-resque": "1.3.6",
"matomo/device-detector": "4.2.3",
@ -70,6 +70,10 @@
{
"type": "git",
"url": "https://github.com/lohanidamodar/audit"
},
{
"type": "git",
"url": "https://github.com/utopia-php/storage"
}
],
"require-dev": {

24
composer.lock generated
View file

@ -4,7 +4,7 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
"This file is @generated automatically"
],
"content-hash": "14172d7cbfa5a5b7353d3f8b24e44b81",
"content-hash": "8c63d8df231b87f77b8ee5b8e10242bc",
"packages": [
{
"name": "adhocore/jwt",
@ -2337,17 +2337,11 @@
},
{
"name": "utopia-php/storage",
"version": "0.5.0",
"version": "dev-feat-large-file-support",
"source": {
"type": "git",
"url": "https://github.com/utopia-php/storage.git",
"reference": "92ae20c7a2ac329f573a58a82dc245134cc63408"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/utopia-php/storage/zipball/92ae20c7a2ac329f573a58a82dc245134cc63408",
"reference": "92ae20c7a2ac329f573a58a82dc245134cc63408",
"shasum": ""
"url": "https://github.com/utopia-php/storage",
"reference": "2e910d935c29cef80ee71b004bcd20dc2e219501"
},
"require": {
"php": ">=7.4",
@ -2363,7 +2357,6 @@
"Utopia\\Storage\\": "src/Storage"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
@ -2381,11 +2374,7 @@
"upf",
"utopia"
],
"support": {
"issues": "https://github.com/utopia-php/storage/issues",
"source": "https://github.com/utopia-php/storage/tree/0.5.0"
},
"time": "2021-04-15T16:43:12+00:00"
"time": "2021-07-07T08:29:23+00:00"
},
{
"name": "utopia-php/swoole",
@ -6235,7 +6224,8 @@
"minimum-stability": "stable",
"stability-flags": {
"utopia-php/abuse": 20,
"utopia-php/audit": 20
"utopia-php/audit": 20,
"utopia-php/storage": 20
},
"prefer-stable": false,
"prefer-lowest": false,

12
src/Appwrite/Utopia/Response/Model/File.php
View file

@ -60,6 +60,18 @@ class File extends Model
'default' => 0,
'example' => 17890,
])
->addRule('totalChunks', [
'type' => self::TYPE_INTEGER,
'description' => 'Total number of chunks available',
'default' => 0,
'example' => 17890,
])
->addRule('uploadedChunks', [
'type' => self::TYPE_INTEGER,
'description' => 'Total number of chunks uploaded',
'default' => 0,
'example' => 17890,
])
;
}

43
tests/e2e/Services/Storage/StorageBase.php
View file

@ -77,6 +77,49 @@ trait StorageBase
$this->assertEquals('video/mp4', $file2['body']['mimeType']);
$this->assertEquals(23660615, $file2['body']['sizeOriginal']);
$this->assertEquals(md5_file(realpath(__DIR__ . '/../../../resources/disk-a/large-file.mp4')), $file2['body']['signature']); // should validate that the file is not encrypted
/**
* Chunked Upload
*/
$source = __DIR__ . "/../../../resources/disk-a/large-file.mp4";
$totalSize = \filesize($source);
$chunkSize = 5000000;
$start = 0;
$handle = @fopen($source, "rb");
$uploadId = '';
$op = __DIR__ . '/chunk.part';
while ($start < $totalSize) {
$contents = fread($handle, $chunkSize);
$cc = fopen($op, 'wb');
fwrite($cc, $contents);
fclose($cc);
$curlFile = new CURLFile($op, 'video/mp4', 'large-file.mp4');
$contentRanges = 'bytes ' . $start . '-' . min((($start + $chunkSize) - 1), $totalSize) . '/' . $totalSize;
$largeFile = $this->client->call(Client::METHOD_POST, '/storage/buckets/' . $bucket2['body']['$id'] . '/files', array_merge([
'content-type' => 'multipart/form-data',
'x-appwrite-project' => $this->getProject()['$id'],
'content-range' => $contentRanges,
'x-appwrite-upload-id' => $uploadId,
], $this->getHeaders()), [
'file' => $curlFile,
'read' => ['role:all'],
'write' => ['role:all'],
]);
$uploadId = $largeFile['body']['$id'];
$start += strlen($contents);
fseek($handle, $start);
}
\unlink($op);
@fclose($handle);
$this->assertEquals(201, $largeFile['headers']['status-code']);
$this->assertNotEmpty($largeFile['body']['$id']);
$this->assertIsInt($largeFile['body']['dateCreated']);
$this->assertEquals('large-file.mp4', $largeFile['body']['name']);
$this->assertEquals('video/mp4', $largeFile['body']['mimeType']);
$this->assertEquals($totalSize, $largeFile['body']['sizeOriginal']);
$this->assertEquals(md5_file(realpath(__DIR__ . '/../../../resources/disk-a/large-file.mp4')), $largeFile['body']['signature']); // should validate that the file is not encrypted
/**
* Test for FAILURE unknown Bucket