CodeMirror/appwrite
1
0
Fork 0
mirror of synced 2024-10-02 18:26:49 +13:00
Code Issues Projects Releases Wiki Activity

first working POC of large file upload

Browse source
This commit is contained in:
Damodar Lohani 2021-07-07 15:52:11 +05:45
parent 8d40a1b3ce
commit 5db04e09f9
6 changed files with 324 additions and 154 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
app/config/collections2.php
View file

@ -625,6 +625,26 @@ $collections = [
'array' => false,
'filters' => [],
],
[
'$id' => 'totalChunks',
'type' => Database::VAR_INTEGER,
'format' => '',
'size' => 0,
'signed' => false,
'required' => false,
'array' => false,
'filters' => [],
],
[
'$id' => 'uploadedChunks',
'type' => Database::VAR_INTEGER,
'format' => '',
'size' => 0,
'signed' => false,
'required' => false,
'array' => false,
'filters' => [],
],
],
'indexes' => [
[

185
app/controllers/api/storage.php
View file

@ -1,30 +1,30 @@
<?php
use Utopia\App;
use Utopia\Exception;
use Utopia\Validator\ArrayList;
use Utopia\Validator\WhiteList;
use Utopia\Validator\Range;
use Utopia\Validator\Text;
use Utopia\Validator\Boolean;
use Utopia\Validator\HexColor;
use Utopia\Cache\Cache;
use Utopia\Cache\Adapter\Filesystem;
use Appwrite\ClamAV\Network;
use Utopia\Database\Document;
use Appwrite\Database\Validator\UID;
use Utopia\Storage\Storage;
use Utopia\Storage\Validator\File;
use Utopia\Storage\Validator\FileSize;
use Utopia\Storage\Validator\Upload;
use Utopia\Storage\Compression\Algorithms\GZIP;
use Utopia\Image\Image;
use Appwrite\OpenSSL\OpenSSL;
use Appwrite\Utopia\Response;
use Utopia\App;
use Utopia\Cache\Adapter\Filesystem;
use Utopia\Cache\Cache;
use Utopia\Config\Config;
use Utopia\Validator\Integer;
use Utopia\Database\Document;
use Utopia\Database\Query;
use Utopia\Exception;
use Utopia\Image\Image;
use Utopia\Storage\Compression\Algorithms\GZIP;
use Utopia\Storage\Storage;
use Utopia\Storage\Validator\File;
use Utopia\Storage\Validator\FileExt;
use Utopia\Storage\Validator\FileSize;
use Utopia\Storage\Validator\Upload;
use Utopia\Validator\ArrayList;
use Utopia\Validator\Boolean;
use Utopia\Validator\HexColor;
use Utopia\Validator\Integer;
use Utopia\Validator\Range;
use Utopia\Validator\Text;
use Utopia\Validator\WhiteList;
App::post('/v1/storage/buckets')
->desc('Create storage bucket')
@ -306,34 +306,82 @@ App::post('/v1/storage/buckets/:bucketId/files')
}
// Make sure we handle a single file and multiple files the same way
$file['name'] = (\is_array($file['name']) && isset($file['name'][0])) ? $file['name'][0] : $file['name'];
$file['tmp_name'] = (\is_array($file['tmp_name']) && isset($file['tmp_name'][0])) ? $file['tmp_name'][0] : $file['tmp_name'];
$file['size'] = (\is_array($file['size']) && isset($file['size'][0])) ? $file['size'][0] : $file['size'];
$fileName = (\is_array($file['name']) && isset($file['name'][0])) ? $file['name'][0] : $file['name'];
$fileTmpName = (\is_array($file['tmp_name']) && isset($file['tmp_name'][0])) ? $file['tmp_name'][0] : $file['tmp_name'];
$size = (\is_array($file['size']) && isset($file['size'][0])) ? $file['size'][0] : $file['size'];
$contentRange = $request->getHeader('content-range');
$uploadId = \uniqid();
$chunk = 1;
$chunks = 1;
if (!empty($contentRange)) {
$uploadId = empty($request->getHeader('x-appwrite-upload-id')) ? $uploadId : $request->getHeader('x-appwrite-upload-id');
$contentRange = explode(" ", $contentRange);
if (count($contentRange) != 2) {
throw new Exception('Invalid content-range header', 400);
}
$rangeData = explode("/", $contentRange[1]);
if (count($rangeData) != 2) {
throw new Exception('Invalid content-range header', 400);
}
$size = (int) $rangeData[1];
$parts = explode("-", $rangeData[0]);
if (count($parts) != 2) {
throw new Exception('Invalid content-range header', 400);
}
$start = (int) $parts[0];
$end = (int) $parts[1];
if ($start > $end || $end > $size) {
throw new Exception('Invalid content-range header', 400);
}
if ($end == $size) {
$chunks = $chunk = -1;
} else {
$chunks = (int) ceil($size / ($end + 1 - $start));
$chunk = (int) ($start / ($end + 1 - $start));
}
}
// Check if file type is allowed (feature for project settings?)
if (!empty($allowedFileExtensions) && !$fileExt->isValid($file['name'])) {
if (!empty($allowedFileExtensions) && !$fileExt->isValid($fileName)) {
throw new Exception('File extension not allowed', 400);
}
if (!$fileSize->isValid($file['size'])) { // Check if file size is exceeding allowed limit
if (!$fileSize->isValid($size)) { // Check if file size is exceeding allowed limit
throw new Exception('File size not allowed', 400);
}
$device = Storage::getDevice('files');
if (!$upload->isValid($file['tmp_name'])) {
if (!$upload->isValid($fileTmpName)) {
throw new Exception('Invalid file', 403);
}
// Save to storage
$size = $device->getFileSize($file['tmp_name']);
$path = $device->getPath(\uniqid().'.'.\pathinfo($file['name'], PATHINFO_EXTENSION));
$path = $bucket->getId() . '/' . $path;
$size = $size ?? $device->getFileSize($fileTmpName);
$path = $device->getPath($uploadId . '.' . \pathinfo($fileName, PATHINFO_EXTENSION));
$path = $bucket->getId() . $path;
if (!$device->upload($file['tmp_name'], $path)) { // TODO deprecate 'upload' and replace with 'move'
throw new Exception('Failed moving file', 500);
$file = $dbForInternal->getDocument('files', $uploadId);
if (!$file->isEmpty()) {
$chunks = $file->getAttribute('totalChunks', 1);
if ($chunk == -1) {
$chunk = $chunks - 1;
}
}
$uploadedChunks = $device->upload($fileTmpName, $path, $chunk, $chunks);
if (empty($uploadedChunks)) {
throw new Exception('Failed uploading file', 500);
}
if ($uploadedChunks == $chunks) {
$mimeType = $device->getFileMimeType($path); // Get mime-type before compression and encryption
if (App::getEnv('_APP_STORAGE_ANTIVIRUS') === 'enabled' && $bucket->getAttribute('antiVirus', true) && $size <= APP_LIMIT_ANTIVIRUS) {
@ -365,38 +413,91 @@ App::post('/v1/storage/buckets/:bucketId/files')
$sizeActual = $device->getFileSize($path);
$read = (is_null($read) && !$user->isEmpty()) ? ['user:' . $user->getId()] : $read ?? [];
$write = (is_null($write) && !$user->isEmpty()) ? ['user:' . $user->getId()] : $write ?? [];
$algorithm = empty($compressor) ? '' : $compressor->getName();
$fileHash = $device->getFileHash($path);
if ($bucket->getAttribute('encryption', true) && $size <= APP_LIMIT_ENCRYPTION) {
$openSSLVersion = '1';
$openSSLCipher = OpenSSL::CIPHER_AES_128_GCM;
$openSSLTag = \bin2hex($tag);
$openSSLIV = \bin2hex($iv);
}
if ($file->isEmpty()) {
$data = [
'$read' => $read,
'$write' => $write,
'dateCreated' => \time(),
'bucketId' => $bucket->getId(),
'name' => $fileName,
'path' => $path,
'signature' => $fileHash,
'mimeType' => $mimeType,
'sizeOriginal' => $size,
'sizeActual' => $sizeActual,
'algorithm' => $algorithm,
'comment' => '',
'totalChunks' => $chunks,
'uploadedChunks' => $uploadedChunks,
'openSSLVersion' => $openSSLVersion,
'openSSLCipher' => $openSSLCipher,
'openSSLTag' => $openSSLTag,
'openSSLIV' => $openSSLIV,
];
$file = $dbForInternal->createDocument('files', new Document($data));
} else {
$file = $dbForInternal->updateDocument('files', $uploadId, $file
->setAttribute('$read', $read)
->setAttribute('$write', $write)
->setAttribute('signature', $fileHash)
->setAttribute('mimeType', $mimeType)
->setAttribute('sizeActual', $sizeActual)
->setAttribute('algorithm', $algorithm)
->setAttribute('openSSLVersion', $openSSLVersion)
->setAttribute('openSSLCipher', $openSSLCipher)
->setAttribute('openSSLTag', $openSSLTag)
->setAttribute('openSSLIV', $openSSLIV)
);
}
} else {
if ($file->isEmpty()) {
$data = [
'$id' => $uploadId,
'$read' => (is_null($read) && !$user->isEmpty()) ? ['user:' . $user->getId()] : $read ?? [], // By default set read permissions for user
'$write' => (is_null($write) && !$user->isEmpty()) ? ['user:' . $user->getId()] : $write ?? [], // By default set write permissions for user
'dateCreated' => \time(),
'bucketId' => $bucket->getId(),
'name' => $file['name'],
'name' => $fileName,
'path' => $path,
'signature' => $device->getFileHash($path),
'mimeType' => $mimeType,
'signature' => '',
'mimeType' => '',
'sizeOriginal' => $size,
'sizeActual' => $sizeActual,
'algorithm' => empty($compressor) ? '' : $compressor->getName(),
'sizeActual' => 0,
'algorithm' => '',
'comment' => '',
'totalChunks' => $chunks,
'uploadedChunks' => $uploadedChunks,
];
if($bucket->getAttribute('encryption', true) && $size <= APP_LIMIT_ENCRYPTION) {
$data['openSSLVersion'] = '1';
$data['openSSLCipher'] = OpenSSL::CIPHER_AES_128_GCM;
$data['openSSLTag'] = \bin2hex($tag);
$data['openSSLIV'] = \bin2hex($iv);
}
$file = $dbForInternal->createDocument('files', new Document($data));
} else {
$file = $dbForInternal->updateDocument('files', $uploadId, $file
->setAttribute('uploadedChunks', $uploadedChunks)
);
}
}
$audits
->setParam('event', 'storage.files.create')
->setParam('resource', 'storage/files/' . $file->getId())
;
if (!empty($sizeActual)) {
$usage
->setParam('storage', $sizeActual)
;
}
$response->setStatusCode(Response::STATUS_CODE_CREATED);
$response->dynamic2($file, Response::MODEL_FILE);

6
composer.json
View file

@ -51,7 +51,7 @@
"utopia-php/preloader": "0.2.*",
"utopia-php/domains": "1.1.*",
"utopia-php/swoole": "0.2.*",
"utopia-php/storage": "0.5.*",
"utopia-php/storage": "dev-feat-large-file-support",
"utopia-php/image": "0.5.*",
"resque/php-resque": "1.3.6",
"matomo/device-detector": "4.2.3",
@ -70,6 +70,10 @@
{
"type": "git",
"url": "https://github.com/lohanidamodar/audit"
},
{
"type": "git",
"url": "https://github.com/utopia-php/storage"
}
],
"require-dev": {

24
composer.lock generated
View file

@ -4,7 +4,7 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
"This file is @generated automatically"
],
"content-hash": "14172d7cbfa5a5b7353d3f8b24e44b81",
"content-hash": "8c63d8df231b87f77b8ee5b8e10242bc",
"packages": [
{
"name": "adhocore/jwt",
@ -2337,17 +2337,11 @@
},
{
"name": "utopia-php/storage",
"version": "0.5.0",
"version": "dev-feat-large-file-support",
"source": {
"type": "git",
"url": "https://github.com/utopia-php/storage.git",
"reference": "92ae20c7a2ac329f573a58a82dc245134cc63408"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/utopia-php/storage/zipball/92ae20c7a2ac329f573a58a82dc245134cc63408",
"reference": "92ae20c7a2ac329f573a58a82dc245134cc63408",
"shasum": ""
"url": "https://github.com/utopia-php/storage",
"reference": "2e910d935c29cef80ee71b004bcd20dc2e219501"
},
"require": {
"php": ">=7.4",
@ -2363,7 +2357,6 @@
"Utopia\\Storage\\": "src/Storage"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
@ -2381,11 +2374,7 @@
"upf",
"utopia"
],
"support": {
"issues": "https://github.com/utopia-php/storage/issues",
"source": "https://github.com/utopia-php/storage/tree/0.5.0"
},
"time": "2021-04-15T16:43:12+00:00"
"time": "2021-07-07T08:29:23+00:00"
},
{
"name": "utopia-php/swoole",
@ -6235,7 +6224,8 @@
"minimum-stability": "stable",
"stability-flags": {
"utopia-php/abuse": 20,
"utopia-php/audit": 20
"utopia-php/audit": 20,
"utopia-php/storage": 20
},
"prefer-stable": false,
"prefer-lowest": false,

12
src/Appwrite/Utopia/Response/Model/File.php
View file

@ -60,6 +60,18 @@ class File extends Model
'default' => 0,
'example' => 17890,
])
->addRule('totalChunks', [
'type' => self::TYPE_INTEGER,
'description' => 'Total number of chunks available',
'default' => 0,
'example' => 17890,
])
->addRule('uploadedChunks', [
'type' => self::TYPE_INTEGER,
'description' => 'Total number of chunks uploaded',
'default' => 0,
'example' => 17890,
])
;
}

43
tests/e2e/Services/Storage/StorageBase.php
View file

@ -78,6 +78,49 @@ trait StorageBase
$this->assertEquals(23660615, $file2['body']['sizeOriginal']);
$this->assertEquals(md5_file(realpath(__DIR__ . '/../../../resources/disk-a/large-file.mp4')), $file2['body']['signature']); // should validate that the file is not encrypted
/**
* Chunked Upload
*/
$source = __DIR__ . "/../../../resources/disk-a/large-file.mp4";
$totalSize = \filesize($source);
$chunkSize = 5000000;
$start = 0;
$handle = @fopen($source, "rb");
$uploadId = '';
$op = __DIR__ . '/chunk.part';
while ($start < $totalSize) {
$contents = fread($handle, $chunkSize);
$cc = fopen($op, 'wb');
fwrite($cc, $contents);
fclose($cc);
$curlFile = new CURLFile($op, 'video/mp4', 'large-file.mp4');
$contentRanges = 'bytes ' . $start . '-' . min((($start + $chunkSize) - 1), $totalSize) . '/' . $totalSize;
$largeFile = $this->client->call(Client::METHOD_POST, '/storage/buckets/' . $bucket2['body']['$id'] . '/files', array_merge([
'content-type' => 'multipart/form-data',
'x-appwrite-project' => $this->getProject()['$id'],
'content-range' => $contentRanges,
'x-appwrite-upload-id' => $uploadId,
], $this->getHeaders()), [
'file' => $curlFile,
'read' => ['role:all'],
'write' => ['role:all'],
]);
$uploadId = $largeFile['body']['$id'];
$start += strlen($contents);
fseek($handle, $start);
}
\unlink($op);
@fclose($handle);
$this->assertEquals(201, $largeFile['headers']['status-code']);
$this->assertNotEmpty($largeFile['body']['$id']);
$this->assertIsInt($largeFile['body']['dateCreated']);
$this->assertEquals('large-file.mp4', $largeFile['body']['name']);
$this->assertEquals('video/mp4', $largeFile['body']['mimeType']);
$this->assertEquals($totalSize, $largeFile['body']['sizeOriginal']);
$this->assertEquals(md5_file(realpath(__DIR__ . '/../../../resources/disk-a/large-file.mp4')), $largeFile['body']['signature']); // should validate that the file is not encrypted
/**
* Test for FAILURE unknown Bucket
*/