Merge pull request #1665 from lohanidamodar/feat-large-function-tags
Feat-large-function-tags
This commit is contained in:
commit
5564c3c34c
4 changed files with 137 additions and 20 deletions
|
@ -1335,6 +1335,28 @@ $collections = [
|
|||
'array' => false,
|
||||
'filters' => [],
|
||||
],
|
||||
[
|
||||
'$id' => 'chunksTotal',
|
||||
'type' => Database::VAR_INTEGER,
|
||||
'format' => '',
|
||||
'size' => 0,
|
||||
'signed' => false,
|
||||
'required' => false,
|
||||
'default' => null,
|
||||
'array' => false,
|
||||
'filters' => [],
|
||||
],
|
||||
[
|
||||
'$id' => 'chunksUploaded',
|
||||
'type' => Database::VAR_INTEGER,
|
||||
'format' => '',
|
||||
'size' => 0,
|
||||
'signed' => false,
|
||||
'required' => false,
|
||||
'default' => null,
|
||||
'array' => false,
|
||||
'filters' => [],
|
||||
],
|
||||
],
|
||||
'indexes' => [
|
||||
[
|
||||
|
|
|
@ -423,40 +423,100 @@ App::post('/v1/functions/:functionId/tags')
|
|||
}
|
||||
|
||||
// Make sure we handle a single file and multiple files the same way
|
||||
$file['name'] = (\is_array($file['name']) && isset($file['name'][0])) ? $file['name'][0] : $file['name'];
|
||||
$file['tmp_name'] = (\is_array($file['tmp_name']) && isset($file['tmp_name'][0])) ? $file['tmp_name'][0] : $file['tmp_name'];
|
||||
$file['size'] = (\is_array($file['size']) && isset($file['size'][0])) ? $file['size'][0] : $file['size'];
|
||||
$fileName = (\is_array($file['name']) && isset($file['name'][0])) ? $file['name'][0] : $file['name'];
|
||||
$fileTmpName = (\is_array($file['tmp_name']) && isset($file['tmp_name'][0])) ? $file['tmp_name'][0] : $file['tmp_name'];
|
||||
$size = (\is_array($file['size']) && isset($file['size'][0])) ? $file['size'][0] : $file['size'];
|
||||
|
||||
if (!$fileExt->isValid($file['name'])) { // Check if file type is allowed
|
||||
throw new Exception('File type not allowed', 400);
|
||||
}
|
||||
|
||||
if (!$fileSize->isValid($file['size'])) { // Check if file size is exceeding allowed limit
|
||||
$contentRange = $request->getHeader('content-range');
|
||||
$tagId = $dbForInternal->getId();
|
||||
$chunk = 1;
|
||||
$chunks = 1;
|
||||
|
||||
if (!empty($contentRange)) {
|
||||
$start = $request->getContentRangeStart();
|
||||
$end = $request->getContentRangeEnd();
|
||||
$size = $request->getContentRangeSize();
|
||||
$tagId = $request->getHeader('x-appwrite-id', $tagId);
|
||||
if(is_null($start) || is_null($end) || is_null($size)) {
|
||||
throw new Exception('Invalid content-range header', 400);
|
||||
}
|
||||
|
||||
if ($end == $size) {
|
||||
//if it's a last chunks the chunk size might differ, so we set the $chunks and $chunk to notify it's last chunk
|
||||
$chunks = $chunk = -1;
|
||||
} else {
|
||||
// Calculate total number of chunks based on the chunk size i.e ($rangeEnd - $rangeStart)
|
||||
$chunks = (int) ceil($size / ($end + 1 - $start));
|
||||
$chunk = (int) ($start / ($end + 1 - $start));
|
||||
}
|
||||
}
|
||||
|
||||
if (!$fileSize->isValid($size)) { // Check if file size is exceeding allowed limit
|
||||
throw new Exception('File size not allowed', 400);
|
||||
}
|
||||
|
||||
if (!$upload->isValid($file['tmp_name'])) {
|
||||
if (!$upload->isValid($fileTmpName)) {
|
||||
throw new Exception('Invalid file', 403);
|
||||
}
|
||||
|
||||
// Save to storage
|
||||
$size = $device->getFileSize($file['tmp_name']);
|
||||
$path = $device->getPath(\uniqid().'.'.\pathinfo($file['name'], PATHINFO_EXTENSION));
|
||||
$size ??= $device->getFileSize($fileTmpName);
|
||||
$path = $device->getPath($tagId.'.'.\pathinfo($fileName, PATHINFO_EXTENSION));
|
||||
|
||||
if (!$device->upload($file['tmp_name'], $path)) { // TODO deprecate 'upload' and replace with 'move'
|
||||
$tag = $dbForInternal->getDocument('tags', $tagId);
|
||||
|
||||
if(!$tag->isEmpty()) {
|
||||
$chunks = $tag->getAttribute('chunksTotal', 1);
|
||||
if($chunk == -1) {
|
||||
$chunk = $chunks - 1;
|
||||
}
|
||||
}
|
||||
|
||||
$chunksUploaded = $device->upload($fileTmpName, $path, $chunk, $chunks);
|
||||
|
||||
if (empty($chunksUploaded)) {
|
||||
throw new Exception('Failed moving file', 500);
|
||||
}
|
||||
|
||||
$tag = $dbForInternal->createDocument('tags', new Document([
|
||||
'$id' => $dbForInternal->getId(),
|
||||
'$read' => [],
|
||||
'$write' => [],
|
||||
'functionId' => $function->getId(),
|
||||
'dateCreated' => time(),
|
||||
'command' => $command,
|
||||
'path' => $path,
|
||||
'size' => $size,
|
||||
]));
|
||||
if($chunksUploaded == $chunks) {
|
||||
$size = $device->getFileSize($path);
|
||||
|
||||
if ($tag->isEmpty()) {
|
||||
$tag = $dbForInternal->createDocument('tags', new Document([
|
||||
'$id' => $tagId,
|
||||
'$read' => [],
|
||||
'$write' => [],
|
||||
'functionId' => $function->getId(),
|
||||
'dateCreated' => time(),
|
||||
'command' => $command,
|
||||
'path' => $path,
|
||||
'size' => $size,
|
||||
]));
|
||||
} else {
|
||||
$tag = $dbForInternal->updateDocument('tags', $tagId, $tag->setAttribute('size', $size));
|
||||
}
|
||||
} else {
|
||||
if($tag->isEmpty()) {
|
||||
$tag = $dbForInternal->createDocument('tags', new Document([
|
||||
'$id' => $tagId,
|
||||
'$read' => [],
|
||||
'$write' => [],
|
||||
'functionId' => $function->getId(),
|
||||
'dateCreated' => time(),
|
||||
'command' => $command,
|
||||
'path' => $path,
|
||||
'size' => 0,
|
||||
'chunksTotal' => $chunks,
|
||||
'chunksUploaded' => $chunksUploaded,
|
||||
]));
|
||||
} else {
|
||||
$tag = $dbForInternal->updateDocument('tags', $tagId, $tag->setAttribute('chunksUploaded', $chunksUploaded));
|
||||
}
|
||||
}
|
||||
|
||||
$usage
|
||||
->setParam('storage', $tag->getAttribute('size', 0))
|
||||
|
|
|
@ -229,6 +229,41 @@ class FunctionsCustomServerTest extends Scope
|
|||
$this->assertIsInt($tag['body']['dateCreated']);
|
||||
$this->assertEquals('php index.php', $tag['body']['command']);
|
||||
$this->assertGreaterThan(10000, $tag['body']['size']);
|
||||
|
||||
/**
|
||||
* Test for Large Code File SUCCESS
|
||||
*/
|
||||
$source = realpath(__DIR__ . '/../../../resources/functions/php-large.tar.gz');
|
||||
$chunkSize = 5*1024*1024;
|
||||
$handle = @fopen($source, "rb");
|
||||
$mimeType = 'application/x-gzip';
|
||||
$counter = 0;
|
||||
$size = filesize($source);
|
||||
$headers = [
|
||||
'content-type' => 'multipart/form-data',
|
||||
'x-appwrite-project' => $this->getProject()['$id']
|
||||
];
|
||||
$id = '';
|
||||
while (!feof($handle)) {
|
||||
$curlFile = new \CURLFile('data://' . $mimeType . ';base64,' . base64_encode(@fread($handle, $chunkSize)), $mimeType, 'php-large-fx.tar.gz');
|
||||
$headers['content-range'] = 'bytes ' . ($counter * $chunkSize) . '-' . min(((($counter * $chunkSize) + $chunkSize) - 1), $size) . '/' . $size;
|
||||
if(!empty($id)) {
|
||||
$headers['x-appwrite-id'] = $id;
|
||||
}
|
||||
$largeTag = $this->client->call(Client::METHOD_POST, '/functions/'.$data['functionId'].'/tags', array_merge($headers, $this->getHeaders()), [
|
||||
'command' => 'php index.php',
|
||||
'code' => $curlFile,
|
||||
]);
|
||||
$counter++;
|
||||
$id = $largeTag['body']['$id'];
|
||||
}
|
||||
@fclose($handle);
|
||||
|
||||
$this->assertEquals(201, $largeTag['headers']['status-code']);
|
||||
$this->assertNotEmpty($largeTag['body']['$id']);
|
||||
$this->assertIsInt($largeTag['body']['dateCreated']);
|
||||
$this->assertEquals('php index.php', $largeTag['body']['command']);
|
||||
$this->assertGreaterThan(10000, $largeTag['body']['size']);
|
||||
|
||||
/**
|
||||
* Test for FAILURE
|
||||
|
@ -279,9 +314,9 @@ class FunctionsCustomServerTest extends Scope
|
|||
], $this->getHeaders()));
|
||||
|
||||
$this->assertEquals($function['headers']['status-code'], 200);
|
||||
$this->assertEquals($function['body']['sum'], 1);
|
||||
$this->assertEquals($function['body']['sum'], 2);
|
||||
$this->assertIsArray($function['body']['tags']);
|
||||
$this->assertCount(1, $function['body']['tags']);
|
||||
$this->assertCount(2, $function['body']['tags']);
|
||||
|
||||
return $data;
|
||||
}
|
||||
|
|
BIN
tests/resources/functions/php-large.tar.gz
Normal file
BIN
tests/resources/functions/php-large.tar.gz
Normal file
Binary file not shown.
Loading…
Reference in a new issue