diff --git a/app/config/collections2.php b/app/config/collections2.php index 6da491b2e1..66b4097811 100644 --- a/app/config/collections2.php +++ b/app/config/collections2.php @@ -1335,6 +1335,28 @@ $collections = [ 'array' => false, 'filters' => [], ], + [ + '$id' => 'chunksTotal', + 'type' => Database::VAR_INTEGER, + 'format' => '', + 'size' => 0, + 'signed' => false, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'chunksUploaded', + 'type' => Database::VAR_INTEGER, + 'format' => '', + 'size' => 0, + 'signed' => false, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], ], 'indexes' => [ [ diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 2a67c6d4c6..2c587a434b 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -423,40 +423,100 @@ App::post('/v1/functions/:functionId/tags') } // Make sure we handle a single file and multiple files the same way - $file['name'] = (\is_array($file['name']) && isset($file['name'][0])) ? $file['name'][0] : $file['name']; - $file['tmp_name'] = (\is_array($file['tmp_name']) && isset($file['tmp_name'][0])) ? $file['tmp_name'][0] : $file['tmp_name']; - $file['size'] = (\is_array($file['size']) && isset($file['size'][0])) ? $file['size'][0] : $file['size']; + $fileName = (\is_array($file['name']) && isset($file['name'][0])) ? $file['name'][0] : $file['name']; + $fileTmpName = (\is_array($file['tmp_name']) && isset($file['tmp_name'][0])) ? $file['tmp_name'][0] : $file['tmp_name']; + $size = (\is_array($file['size']) && isset($file['size'][0])) ? $file['size'][0] : $file['size']; if (!$fileExt->isValid($file['name'])) { // Check if file type is allowed throw new Exception('File type not allowed', 400); } - if (!$fileSize->isValid($file['size'])) { // Check if file size is exceeding allowed limit + $contentRange = $request->getHeader('content-range'); + $tagId = $dbForInternal->getId(); + $chunk = 1; + $chunks = 1; + + if (!empty($contentRange)) { + $start = $request->getContentRangeStart(); + $end = $request->getContentRangeEnd(); + $size = $request->getContentRangeSize(); + $tagId = $request->getHeader('x-appwrite-id', $tagId); + if(is_null($start) || is_null($end) || is_null($size)) { + throw new Exception('Invalid content-range header', 400); + } + + if ($end == $size) { + //if it's a last chunks the chunk size might differ, so we set the $chunks and $chunk to notify it's last chunk + $chunks = $chunk = -1; + } else { + // Calculate total number of chunks based on the chunk size i.e ($rangeEnd - $rangeStart) + $chunks = (int) ceil($size / ($end + 1 - $start)); + $chunk = (int) ($start / ($end + 1 - $start)); + } + } + + if (!$fileSize->isValid($size)) { // Check if file size is exceeding allowed limit throw new Exception('File size not allowed', 400); } - if (!$upload->isValid($file['tmp_name'])) { + if (!$upload->isValid($fileTmpName)) { throw new Exception('Invalid file', 403); } // Save to storage - $size = $device->getFileSize($file['tmp_name']); - $path = $device->getPath(\uniqid().'.'.\pathinfo($file['name'], PATHINFO_EXTENSION)); + $size ??= $device->getFileSize($fileTmpName); + $path = $device->getPath($tagId.'.'.\pathinfo($fileName, PATHINFO_EXTENSION)); - if (!$device->upload($file['tmp_name'], $path)) { // TODO deprecate 'upload' and replace with 'move' + $tag = $dbForInternal->getDocument('tags', $tagId); + + if(!$tag->isEmpty()) { + $chunks = $tag->getAttribute('chunksTotal', 1); + if($chunk == -1) { + $chunk = $chunks - 1; + } + } + + $chunksUploaded = $device->upload($fileTmpName, $path, $chunk, $chunks); + + if (empty($chunksUploaded)) { throw new Exception('Failed moving file', 500); } - $tag = $dbForInternal->createDocument('tags', new Document([ - '$id' => $dbForInternal->getId(), - '$read' => [], - '$write' => [], - 'functionId' => $function->getId(), - 'dateCreated' => time(), - 'command' => $command, - 'path' => $path, - 'size' => $size, - ])); + if($chunksUploaded == $chunks) { + $size = $device->getFileSize($path); + + if ($tag->isEmpty()) { + $tag = $dbForInternal->createDocument('tags', new Document([ + '$id' => $tagId, + '$read' => [], + '$write' => [], + 'functionId' => $function->getId(), + 'dateCreated' => time(), + 'command' => $command, + 'path' => $path, + 'size' => $size, + ])); + } else { + $tag = $dbForInternal->updateDocument('tags', $tagId, $tag->setAttribute('size', $size)); + } + } else { + if($tag->isEmpty()) { + $tag = $dbForInternal->createDocument('tags', new Document([ + '$id' => $tagId, + '$read' => [], + '$write' => [], + 'functionId' => $function->getId(), + 'dateCreated' => time(), + 'command' => $command, + 'path' => $path, + 'size' => 0, + 'chunksTotal' => $chunks, + 'chunksUploaded' => $chunksUploaded, + ])); + } else { + $tag = $dbForInternal->updateDocument('tags', $tagId, $tag->setAttribute('chunksUploaded', $chunksUploaded)); + } + } $usage ->setParam('storage', $tag->getAttribute('size', 0)) diff --git a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php index cb820b45c3..dfc5855664 100644 --- a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php +++ b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php @@ -229,6 +229,41 @@ class FunctionsCustomServerTest extends Scope $this->assertIsInt($tag['body']['dateCreated']); $this->assertEquals('php index.php', $tag['body']['command']); $this->assertGreaterThan(10000, $tag['body']['size']); + + /** + * Test for Large Code File SUCCESS + */ + $source = realpath(__DIR__ . '/../../../resources/functions/php-large.tar.gz'); + $chunkSize = 5*1024*1024; + $handle = @fopen($source, "rb"); + $mimeType = 'application/x-gzip'; + $counter = 0; + $size = filesize($source); + $headers = [ + 'content-type' => 'multipart/form-data', + 'x-appwrite-project' => $this->getProject()['$id'] + ]; + $id = ''; + while (!feof($handle)) { + $curlFile = new \CURLFile('data://' . $mimeType . ';base64,' . base64_encode(@fread($handle, $chunkSize)), $mimeType, 'php-large-fx.tar.gz'); + $headers['content-range'] = 'bytes ' . ($counter * $chunkSize) . '-' . min(((($counter * $chunkSize) + $chunkSize) - 1), $size) . '/' . $size; + if(!empty($id)) { + $headers['x-appwrite-id'] = $id; + } + $largeTag = $this->client->call(Client::METHOD_POST, '/functions/'.$data['functionId'].'/tags', array_merge($headers, $this->getHeaders()), [ + 'command' => 'php index.php', + 'code' => $curlFile, + ]); + $counter++; + $id = $largeTag['body']['$id']; + } + @fclose($handle); + + $this->assertEquals(201, $largeTag['headers']['status-code']); + $this->assertNotEmpty($largeTag['body']['$id']); + $this->assertIsInt($largeTag['body']['dateCreated']); + $this->assertEquals('php index.php', $largeTag['body']['command']); + $this->assertGreaterThan(10000, $largeTag['body']['size']); /** * Test for FAILURE @@ -279,9 +314,9 @@ class FunctionsCustomServerTest extends Scope ], $this->getHeaders())); $this->assertEquals($function['headers']['status-code'], 200); - $this->assertEquals($function['body']['sum'], 1); + $this->assertEquals($function['body']['sum'], 2); $this->assertIsArray($function['body']['tags']); - $this->assertCount(1, $function['body']['tags']); + $this->assertCount(2, $function['body']['tags']); return $data; } diff --git a/tests/resources/functions/php-large.tar.gz b/tests/resources/functions/php-large.tar.gz new file mode 100644 index 0000000000..7ca5857029 Binary files /dev/null and b/tests/resources/functions/php-large.tar.gz differ