Merge pull request #1665 from lohanidamodar/feat-large-function-tags

Feat-large-function-tags

app/config/collections2.php

@@ -1335,6 +1335,28 @@ $collections = [
                 'array' => false,
                 'filters' => [],
             ],
+            [
+                '$id' => 'chunksTotal',
+                'type' => Database::VAR_INTEGER,
+                'format' => '',
+                'size' => 0,
+                'signed' => false,
+                'required' => false,
+                'default' => null,
+                'array' => false,
+                'filters' => [],
+            ],
+            [
+                '$id' => 'chunksUploaded',
+                'type' => Database::VAR_INTEGER,
+                'format' => '',
+                'size' => 0,
+                'signed' => false,
+                'required' => false,
+                'default' => null,
+                'array' => false,
+                'filters' => [],
+            ],
         ],
         'indexes' => [
             [

app/controllers/api/functions.php

@@ -423,40 +423,100 @@ App::post('/v1/functions/:functionId/tags')
         }
 
         // Make sure we handle a single file and multiple files the same way
-        $file['name'] = (\is_array($file['name']) && isset($file['name'][0])) ? $file['name'][0] : $file['name'];
-        $file['tmp_name'] = (\is_array($file['tmp_name']) && isset($file['tmp_name'][0])) ? $file['tmp_name'][0] : $file['tmp_name'];
-        $file['size'] = (\is_array($file['size']) && isset($file['size'][0])) ? $file['size'][0] : $file['size'];
+        $fileName = (\is_array($file['name']) && isset($file['name'][0])) ? $file['name'][0] : $file['name'];
+        $fileTmpName = (\is_array($file['tmp_name']) && isset($file['tmp_name'][0])) ? $file['tmp_name'][0] : $file['tmp_name'];
+        $size = (\is_array($file['size']) && isset($file['size'][0])) ? $file['size'][0] : $file['size'];
 
         if (!$fileExt->isValid($file['name'])) { // Check if file type is allowed
             throw new Exception('File type not allowed', 400);
         }
 
-        if (!$fileSize->isValid($file['size'])) { // Check if file size is exceeding allowed limit
+        $contentRange = $request->getHeader('content-range');
+        $tagId = $dbForInternal->getId();
+        $chunk = 1;
+        $chunks = 1;
+
+        if (!empty($contentRange)) {
+            $start = $request->getContentRangeStart();
+            $end = $request->getContentRangeEnd();
+            $size = $request->getContentRangeSize();
+            $tagId = $request->getHeader('x-appwrite-id', $tagId);
+            if(is_null($start) || is_null($end) || is_null($size)) {
+                throw new Exception('Invalid content-range header', 400);
+            }
+
+            if ($end == $size) {
+                //if it's a last chunks the chunk size might differ, so we set the $chunks and $chunk to notify it's last chunk
+                $chunks = $chunk = -1;
+            } else {
+                // Calculate total number of chunks based on the chunk size i.e ($rangeEnd - $rangeStart)
+                $chunks = (int) ceil($size / ($end + 1 - $start));
+                $chunk = (int) ($start / ($end + 1 - $start));
+            }
+        }
+
+        if (!$fileSize->isValid($size)) { // Check if file size is exceeding allowed limit
             throw new Exception('File size not allowed', 400);
         }
 
-        if (!$upload->isValid($file['tmp_name'])) {
+        if (!$upload->isValid($fileTmpName)) {
             throw new Exception('Invalid file', 403);
         }
 
         // Save to storage
-        $size = $device->getFileSize($file['tmp_name']);
-        $path = $device->getPath(\uniqid().'.'.\pathinfo($file['name'], PATHINFO_EXTENSION));
+        $size ??= $device->getFileSize($fileTmpName);
+        $path = $device->getPath($tagId.'.'.\pathinfo($fileName, PATHINFO_EXTENSION));
         
-        if (!$device->upload($file['tmp_name'], $path)) { // TODO deprecate 'upload' and replace with 'move'
+        $tag = $dbForInternal->getDocument('tags', $tagId);
+
+        if(!$tag->isEmpty()) {
+            $chunks = $tag->getAttribute('chunksTotal', 1);
+            if($chunk == -1) {
+                $chunk = $chunks - 1;
+            }
+        }
+
+        $chunksUploaded = $device->upload($fileTmpName, $path, $chunk, $chunks);
+
+        if (empty($chunksUploaded)) {
             throw new Exception('Failed moving file', 500);
         }
         
-        $tag = $dbForInternal->createDocument('tags', new Document([
-            '$id' => $dbForInternal->getId(),
-            '$read' => [],
-            '$write' => [],
-            'functionId' => $function->getId(),
-            'dateCreated' => time(),
-            'command' => $command,
-            'path' => $path,
-            'size' => $size,
-        ]));
+        if($chunksUploaded == $chunks) {
+            $size = $device->getFileSize($path);
+
+            if ($tag->isEmpty()) {
+                $tag = $dbForInternal->createDocument('tags', new Document([
+                    '$id' => $tagId,
+                    '$read' => [],
+                    '$write' => [],
+                    'functionId' => $function->getId(),
+                    'dateCreated' => time(),
+                    'command' => $command,
+                    'path' => $path,
+                    'size' => $size,
+                ]));
+            } else {
+                $tag = $dbForInternal->updateDocument('tags', $tagId, $tag->setAttribute('size', $size));
+            }
+        } else {
+            if($tag->isEmpty()) {
+                $tag = $dbForInternal->createDocument('tags', new Document([
+                    '$id' => $tagId,
+                    '$read' => [],
+                    '$write' => [],
+                    'functionId' => $function->getId(),
+                    'dateCreated' => time(),
+                    'command' => $command,
+                    'path' => $path,
+                    'size' => 0,
+                    'chunksTotal' => $chunks,
+                    'chunksUploaded' => $chunksUploaded,
+                ]));
+            } else {
+                $tag = $dbForInternal->updateDocument('tags', $tagId, $tag->setAttribute('chunksUploaded', $chunksUploaded));
+            }
+        }
 
         $usage
             ->setParam('storage', $tag->getAttribute('size', 0))

tests/e2e/Services/Functions/FunctionsCustomServerTest.php

@@ -229,6 +229,41 @@ class FunctionsCustomServerTest extends Scope
         $this->assertIsInt($tag['body']['dateCreated']);
         $this->assertEquals('php index.php', $tag['body']['command']);
         $this->assertGreaterThan(10000, $tag['body']['size']);
+
+        /**
+         * Test for Large Code File SUCCESS
+         */
+        $source = realpath(__DIR__ . '/../../../resources/functions/php-large.tar.gz');
+        $chunkSize = 5*1024*1024;
+        $handle = @fopen($source, "rb");
+        $mimeType = 'application/x-gzip';
+        $counter = 0;
+        $size = filesize($source);
+        $headers = [
+            'content-type' => 'multipart/form-data',
+            'x-appwrite-project' => $this->getProject()['$id']
+        ];
+        $id = '';
+        while (!feof($handle)) {
+            $curlFile = new \CURLFile('data://' . $mimeType . ';base64,' . base64_encode(@fread($handle, $chunkSize)), $mimeType, 'php-large-fx.tar.gz');
+            $headers['content-range'] = 'bytes ' . ($counter * $chunkSize) . '-' . min(((($counter * $chunkSize) + $chunkSize) - 1), $size) . '/' . $size;
+            if(!empty($id)) {
+                $headers['x-appwrite-id'] = $id;
+            }
+            $largeTag = $this->client->call(Client::METHOD_POST, '/functions/'.$data['functionId'].'/tags', array_merge($headers, $this->getHeaders()), [
+                'command' => 'php index.php',
+                'code' => $curlFile,
+            ]);
+            $counter++;
+            $id = $largeTag['body']['$id'];
+        }
+        @fclose($handle);
+
+        $this->assertEquals(201, $largeTag['headers']['status-code']);
+        $this->assertNotEmpty($largeTag['body']['$id']);
+        $this->assertIsInt($largeTag['body']['dateCreated']);
+        $this->assertEquals('php index.php', $largeTag['body']['command']);
+        $this->assertGreaterThan(10000, $largeTag['body']['size']);
        
         /**
          * Test for FAILURE
@@ -279,9 +314,9 @@ class FunctionsCustomServerTest extends Scope
         ], $this->getHeaders()));
 
         $this->assertEquals($function['headers']['status-code'], 200);
-        $this->assertEquals($function['body']['sum'], 1);
+        $this->assertEquals($function['body']['sum'], 2);
         $this->assertIsArray($function['body']['tags']);
-        $this->assertCount(1, $function['body']['tags']);
+        $this->assertCount(2, $function['body']['tags']);
 
         return $data;
     }