Merge pull request #1665 from lohanidamodar/feat-large-function-tags
Feat-large-function-tags
This commit is contained in:
commit
5564c3c34c
4 changed files with 137 additions and 20 deletions
|
@ -1335,6 +1335,28 @@ $collections = [
|
||||||
'array' => false,
|
'array' => false,
|
||||||
'filters' => [],
|
'filters' => [],
|
||||||
],
|
],
|
||||||
|
[
|
||||||
|
'$id' => 'chunksTotal',
|
||||||
|
'type' => Database::VAR_INTEGER,
|
||||||
|
'format' => '',
|
||||||
|
'size' => 0,
|
||||||
|
'signed' => false,
|
||||||
|
'required' => false,
|
||||||
|
'default' => null,
|
||||||
|
'array' => false,
|
||||||
|
'filters' => [],
|
||||||
|
],
|
||||||
|
[
|
||||||
|
'$id' => 'chunksUploaded',
|
||||||
|
'type' => Database::VAR_INTEGER,
|
||||||
|
'format' => '',
|
||||||
|
'size' => 0,
|
||||||
|
'signed' => false,
|
||||||
|
'required' => false,
|
||||||
|
'default' => null,
|
||||||
|
'array' => false,
|
||||||
|
'filters' => [],
|
||||||
|
],
|
||||||
],
|
],
|
||||||
'indexes' => [
|
'indexes' => [
|
||||||
[
|
[
|
||||||
|
|
|
@ -423,32 +423,71 @@ App::post('/v1/functions/:functionId/tags')
|
||||||
}
|
}
|
||||||
|
|
||||||
// Make sure we handle a single file and multiple files the same way
|
// Make sure we handle a single file and multiple files the same way
|
||||||
$file['name'] = (\is_array($file['name']) && isset($file['name'][0])) ? $file['name'][0] : $file['name'];
|
$fileName = (\is_array($file['name']) && isset($file['name'][0])) ? $file['name'][0] : $file['name'];
|
||||||
$file['tmp_name'] = (\is_array($file['tmp_name']) && isset($file['tmp_name'][0])) ? $file['tmp_name'][0] : $file['tmp_name'];
|
$fileTmpName = (\is_array($file['tmp_name']) && isset($file['tmp_name'][0])) ? $file['tmp_name'][0] : $file['tmp_name'];
|
||||||
$file['size'] = (\is_array($file['size']) && isset($file['size'][0])) ? $file['size'][0] : $file['size'];
|
$size = (\is_array($file['size']) && isset($file['size'][0])) ? $file['size'][0] : $file['size'];
|
||||||
|
|
||||||
if (!$fileExt->isValid($file['name'])) { // Check if file type is allowed
|
if (!$fileExt->isValid($file['name'])) { // Check if file type is allowed
|
||||||
throw new Exception('File type not allowed', 400);
|
throw new Exception('File type not allowed', 400);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!$fileSize->isValid($file['size'])) { // Check if file size is exceeding allowed limit
|
$contentRange = $request->getHeader('content-range');
|
||||||
|
$tagId = $dbForInternal->getId();
|
||||||
|
$chunk = 1;
|
||||||
|
$chunks = 1;
|
||||||
|
|
||||||
|
if (!empty($contentRange)) {
|
||||||
|
$start = $request->getContentRangeStart();
|
||||||
|
$end = $request->getContentRangeEnd();
|
||||||
|
$size = $request->getContentRangeSize();
|
||||||
|
$tagId = $request->getHeader('x-appwrite-id', $tagId);
|
||||||
|
if(is_null($start) || is_null($end) || is_null($size)) {
|
||||||
|
throw new Exception('Invalid content-range header', 400);
|
||||||
|
}
|
||||||
|
|
||||||
|
if ($end == $size) {
|
||||||
|
//if it's a last chunks the chunk size might differ, so we set the $chunks and $chunk to notify it's last chunk
|
||||||
|
$chunks = $chunk = -1;
|
||||||
|
} else {
|
||||||
|
// Calculate total number of chunks based on the chunk size i.e ($rangeEnd - $rangeStart)
|
||||||
|
$chunks = (int) ceil($size / ($end + 1 - $start));
|
||||||
|
$chunk = (int) ($start / ($end + 1 - $start));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!$fileSize->isValid($size)) { // Check if file size is exceeding allowed limit
|
||||||
throw new Exception('File size not allowed', 400);
|
throw new Exception('File size not allowed', 400);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!$upload->isValid($file['tmp_name'])) {
|
if (!$upload->isValid($fileTmpName)) {
|
||||||
throw new Exception('Invalid file', 403);
|
throw new Exception('Invalid file', 403);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Save to storage
|
// Save to storage
|
||||||
$size = $device->getFileSize($file['tmp_name']);
|
$size ??= $device->getFileSize($fileTmpName);
|
||||||
$path = $device->getPath(\uniqid().'.'.\pathinfo($file['name'], PATHINFO_EXTENSION));
|
$path = $device->getPath($tagId.'.'.\pathinfo($fileName, PATHINFO_EXTENSION));
|
||||||
|
|
||||||
if (!$device->upload($file['tmp_name'], $path)) { // TODO deprecate 'upload' and replace with 'move'
|
$tag = $dbForInternal->getDocument('tags', $tagId);
|
||||||
|
|
||||||
|
if(!$tag->isEmpty()) {
|
||||||
|
$chunks = $tag->getAttribute('chunksTotal', 1);
|
||||||
|
if($chunk == -1) {
|
||||||
|
$chunk = $chunks - 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
$chunksUploaded = $device->upload($fileTmpName, $path, $chunk, $chunks);
|
||||||
|
|
||||||
|
if (empty($chunksUploaded)) {
|
||||||
throw new Exception('Failed moving file', 500);
|
throw new Exception('Failed moving file', 500);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if($chunksUploaded == $chunks) {
|
||||||
|
$size = $device->getFileSize($path);
|
||||||
|
|
||||||
|
if ($tag->isEmpty()) {
|
||||||
$tag = $dbForInternal->createDocument('tags', new Document([
|
$tag = $dbForInternal->createDocument('tags', new Document([
|
||||||
'$id' => $dbForInternal->getId(),
|
'$id' => $tagId,
|
||||||
'$read' => [],
|
'$read' => [],
|
||||||
'$write' => [],
|
'$write' => [],
|
||||||
'functionId' => $function->getId(),
|
'functionId' => $function->getId(),
|
||||||
|
@ -457,6 +496,27 @@ App::post('/v1/functions/:functionId/tags')
|
||||||
'path' => $path,
|
'path' => $path,
|
||||||
'size' => $size,
|
'size' => $size,
|
||||||
]));
|
]));
|
||||||
|
} else {
|
||||||
|
$tag = $dbForInternal->updateDocument('tags', $tagId, $tag->setAttribute('size', $size));
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
if($tag->isEmpty()) {
|
||||||
|
$tag = $dbForInternal->createDocument('tags', new Document([
|
||||||
|
'$id' => $tagId,
|
||||||
|
'$read' => [],
|
||||||
|
'$write' => [],
|
||||||
|
'functionId' => $function->getId(),
|
||||||
|
'dateCreated' => time(),
|
||||||
|
'command' => $command,
|
||||||
|
'path' => $path,
|
||||||
|
'size' => 0,
|
||||||
|
'chunksTotal' => $chunks,
|
||||||
|
'chunksUploaded' => $chunksUploaded,
|
||||||
|
]));
|
||||||
|
} else {
|
||||||
|
$tag = $dbForInternal->updateDocument('tags', $tagId, $tag->setAttribute('chunksUploaded', $chunksUploaded));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
$usage
|
$usage
|
||||||
->setParam('storage', $tag->getAttribute('size', 0))
|
->setParam('storage', $tag->getAttribute('size', 0))
|
||||||
|
|
|
@ -230,6 +230,41 @@ class FunctionsCustomServerTest extends Scope
|
||||||
$this->assertEquals('php index.php', $tag['body']['command']);
|
$this->assertEquals('php index.php', $tag['body']['command']);
|
||||||
$this->assertGreaterThan(10000, $tag['body']['size']);
|
$this->assertGreaterThan(10000, $tag['body']['size']);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Test for Large Code File SUCCESS
|
||||||
|
*/
|
||||||
|
$source = realpath(__DIR__ . '/../../../resources/functions/php-large.tar.gz');
|
||||||
|
$chunkSize = 5*1024*1024;
|
||||||
|
$handle = @fopen($source, "rb");
|
||||||
|
$mimeType = 'application/x-gzip';
|
||||||
|
$counter = 0;
|
||||||
|
$size = filesize($source);
|
||||||
|
$headers = [
|
||||||
|
'content-type' => 'multipart/form-data',
|
||||||
|
'x-appwrite-project' => $this->getProject()['$id']
|
||||||
|
];
|
||||||
|
$id = '';
|
||||||
|
while (!feof($handle)) {
|
||||||
|
$curlFile = new \CURLFile('data://' . $mimeType . ';base64,' . base64_encode(@fread($handle, $chunkSize)), $mimeType, 'php-large-fx.tar.gz');
|
||||||
|
$headers['content-range'] = 'bytes ' . ($counter * $chunkSize) . '-' . min(((($counter * $chunkSize) + $chunkSize) - 1), $size) . '/' . $size;
|
||||||
|
if(!empty($id)) {
|
||||||
|
$headers['x-appwrite-id'] = $id;
|
||||||
|
}
|
||||||
|
$largeTag = $this->client->call(Client::METHOD_POST, '/functions/'.$data['functionId'].'/tags', array_merge($headers, $this->getHeaders()), [
|
||||||
|
'command' => 'php index.php',
|
||||||
|
'code' => $curlFile,
|
||||||
|
]);
|
||||||
|
$counter++;
|
||||||
|
$id = $largeTag['body']['$id'];
|
||||||
|
}
|
||||||
|
@fclose($handle);
|
||||||
|
|
||||||
|
$this->assertEquals(201, $largeTag['headers']['status-code']);
|
||||||
|
$this->assertNotEmpty($largeTag['body']['$id']);
|
||||||
|
$this->assertIsInt($largeTag['body']['dateCreated']);
|
||||||
|
$this->assertEquals('php index.php', $largeTag['body']['command']);
|
||||||
|
$this->assertGreaterThan(10000, $largeTag['body']['size']);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Test for FAILURE
|
* Test for FAILURE
|
||||||
*/
|
*/
|
||||||
|
@ -279,9 +314,9 @@ class FunctionsCustomServerTest extends Scope
|
||||||
], $this->getHeaders()));
|
], $this->getHeaders()));
|
||||||
|
|
||||||
$this->assertEquals($function['headers']['status-code'], 200);
|
$this->assertEquals($function['headers']['status-code'], 200);
|
||||||
$this->assertEquals($function['body']['sum'], 1);
|
$this->assertEquals($function['body']['sum'], 2);
|
||||||
$this->assertIsArray($function['body']['tags']);
|
$this->assertIsArray($function['body']['tags']);
|
||||||
$this->assertCount(1, $function['body']['tags']);
|
$this->assertCount(2, $function['body']['tags']);
|
||||||
|
|
||||||
return $data;
|
return $data;
|
||||||
}
|
}
|
||||||
|
|
BIN
tests/resources/functions/php-large.tar.gz
Normal file
BIN
tests/resources/functions/php-large.tar.gz
Normal file
Binary file not shown.
Loading…
Reference in a new issue