1
0
Fork 0
mirror of synced 2024-05-18 19:52:39 +12:00

Merge pull request #1420 from PineappleIOnic/orchestration-api-test

Implement Utopia Orchestration library into Appwrite
This commit is contained in:
Eldad A. Fux 2021-09-01 00:04:51 +03:00 committed by GitHub
commit 5f0f932428
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 216 additions and 139 deletions

View file

@ -1,9 +1,15 @@
# Version 0.10.0 (Not Released Yet)
- Switch from using Docker CLI to Docker API by intergrating [utopia-php/orchestration](https://github.com/utopia-php/orchestration)
- Added DOCKERHUB_PULL_USERNAME, DOCKERHUB_PULL_PASSWORD and DOCKERHUB_PULL_EMAIL env variables for pulling from
private DockerHub repos
# Version 0.9.4
## Security
- Fixed security vulnerability that exposes project ID's from other admin users (#1453)
# Version 0.9.3
## Bugs

View file

@ -462,6 +462,33 @@ return [
'question' => '',
'filter' => ''
],
[
'name' => 'DOCKERHUB_PULL_USERNAME',
'description' => 'The username for hub.docker.com. This variable is used to pull images from hub.docker.com.',
'introduction' => '0.10.0',
'default' => '',
'required' => false,
'question' => '',
'filter' => ''
],
[
'name' => 'DOCKERHUB_PULL_PASSWORD',
'description' => 'The password for hub.docker.com. This variable is used to pull images from hub.docker.com.',
'introduction' => '0.10.0',
'default' => '',
'required' => false,
'question' => '',
'filter' => ''
],
[
'name' => 'DOCKERHUB_PULL_EMAIL',
'description' => 'The email for hub.docker.com. This variable is used to pull images from hub.docker.com.',
'introduction' => '0.10.0',
'default' => '',
'required' => false,
'question' => '',
'filter' => ''
],
],
[
'category' => 'Maintenance',

View file

@ -13,6 +13,11 @@ use Swoole\Runtime;
use Utopia\App;
use Utopia\CLI\Console;
use Utopia\Config\Config;
use Utopia\Orchestration\Orchestration;
use Utopia\Orchestration\Adapter\DockerAPI;
use Utopia\Orchestration\Container;
use Utopia\Orchestration\Exception\Orchestration as OrchestrationException;
use Utopia\Orchestration\Exception\Timeout as TimeoutException;
require_once __DIR__.'/../workers.php';
@ -23,39 +28,27 @@ Console::success(APP_NAME.' functions worker v1 has started');
$runtimes = Config::getParam('runtimes');
$dockerUser = App::getEnv('DOCKERHUB_PULL_USERNAME', null);
$dockerPass = App::getEnv('DOCKERHUB_PULL_PASSWORD', null);
$dockerEmail = App::getEnv('DOCKERHUB_PULL_EMAIL', null);
$orchestration = new Orchestration(new DockerAPI($dockerUser, $dockerPass, $dockerEmail));
/**
* Warmup Docker Images
*/
$warmupStart = \microtime(true);
Co\run(function() use ($runtimes) { // Warmup: make sure images are ready to run fast 🚀
$dockerUser = App::getEnv('DOCKERHUB_PULL_USERNAME', null);
$dockerPass = App::getEnv('DOCKERHUB_PULL_PASSWORD', null);
if($dockerUser) {
$stdout = '';
$stderr = '';
Console::execute('docker login --username '.$dockerUser.' --password-stdin', $dockerPass, $stdout, $stderr);
Console::log('Docker Login'. $stdout.$stderr);
}
Co\run(function() use ($runtimes, $orchestration) { // Warmup: make sure images are ready to run fast 🚀
foreach($runtimes as $runtime) {
go(function() use ($runtime) {
$stdout = '';
$stderr = '';
go(function() use ($runtime, $orchestration) {
Console::info('Warming up '.$runtime['name'].' '.$runtime['version'].' environment...');
Console::execute('docker pull '.$runtime['image'], '', $stdout, $stderr);
if(!empty($stdout)) {
Console::log($stdout);
}
if(!empty($stderr)) {
Console::error($stderr);
$response = $orchestration->pull($runtime['image']);
if ($response) {
Console::success("Successfully Warmed up {$runtime['name']} {$runtime['version']}!");
} else {
Console::error("Failed to Warmup {$runtime['name']} {$runtime['version']}!");
}
});
}
@ -74,40 +67,17 @@ $stderr = '';
$executionStart = \microtime(true);
$exitCode = Console::execute('docker ps --all --format "name={{.Names}}&status={{.Status}}&labels={{.Labels}}" --filter label=appwrite-type=function'
, '', $stdout, $stderr, 30);
$response = $orchestration->list(['label' => 'appwrite-type=function']);
$list = [];
foreach ($response as $value) {
$list[$value->getName()] = $value;
}
$executionEnd = \microtime(true);
$list = [];
$stdout = \explode("\n", $stdout);
\array_map(function($value) use (&$list) {
$container = [];
\parse_str($value, $container);
if(isset($container['name'])) {
$container = [
'name' => $container['name'],
'online' => (\substr($container['status'], 0, 2) === 'Up'),
'status' => $container['status'],
'labels' => $container['labels'],
];
\array_map(function($value) use (&$container) {
$value = \explode('=', $value);
if(isset($value[0]) && isset($value[1])) {
$container[$value[0]] = $value[1];
}
}, \explode(',', $container['labels']));
$list[$container['name']] = $container;
}
}, $stdout);
Console::info(count($list)." functions listed in " . ($executionEnd - $executionStart) . " seconds with exit code {$exitCode}");
Console::info(count($list).' functions listed in ' . ($executionEnd - $executionStart) . ' seconds');
/**
* 1. Get event args - DONE
@ -297,6 +267,7 @@ class FunctionsV1 extends Worker
public function execute(string $trigger, string $projectId, string $executionId, Database $database, Document $function, string $event = '', string $eventData = '', string $data = '', array $webhooks = [], string $userId = '', string $jwt = ''): void
{
global $list;
global $orchestration;
$runtimes = Config::getParam('runtimes');
@ -355,12 +326,6 @@ class FunctionsV1 extends Worker
'APPWRITE_FUNCTION_PROJECT_ID' => $projectId,
]);
\array_walk($vars, function (&$value, $key) {
$key = $this->filterEnvKey($key);
$value = \escapeshellarg((empty($value)) ? '' : $value);
$value = "--env {$key}={$value}";
});
$tagPath = $tag->getAttribute('path', '');
$tagPathTarget = '/tmp/project-'.$projectId.'/'.$tag->getId().'/code.tar.gz';
$tagPathTargetDir = \pathinfo($tagPathTarget, PATHINFO_DIRNAME);
@ -383,12 +348,14 @@ class FunctionsV1 extends Worker
}
}
if(isset($list[$container]) && !$list[$container]['online']) { // Remove conatiner if not online
if(isset($list[$container]) && !(\substr($list[$container]->getStatus(), 0, 2) === 'Up')) { // Remove conatiner if not online
$stdout = '';
$stderr = '';
if(Console::execute("docker rm {$container}", '', $stdout, $stderr, 30) !== 0) {
throw new Exception('Failed to remove offline container: '.$stderr);
try {
$orchestration->remove($container);
} catch (Exception $e) {
Console::warning('Failed to remove container: '.$e->getMessage());
}
unset($list[$container]);
@ -409,79 +376,102 @@ class FunctionsV1 extends Worker
$executionStart = \microtime(true);
$executionTime = \time();
$cpus = App::getEnv('_APP_FUNCTIONS_CPUS', '');
$memory = App::getEnv('_APP_FUNCTIONS_MEMORY', '');
$swap = App::getEnv('_APP_FUNCTIONS_MEMORY_SWAP', '');
$exitCode = Console::execute("docker run ".
" -d".
" --entrypoint=\"\"".
(empty($cpus) ? "" : (" --cpus=".$cpus)).
(empty($memory) ? "" : (" --memory=".$memory."m")).
(empty($swap) ? "" : (" --memory-swap=".$swap."m")).
" --name={$container}".
" --label appwrite-type=function".
" --label appwrite-created={$executionTime}".
" --volume {$tagPathTargetDir}:/tmp:rw".
" --workdir /usr/local/src".
" ".\implode(" ", $vars).
" {$runtime['image']}".
" tail -f /dev/null"
, '', $stdout, $stderr, 30);
if($exitCode !== 0) {
throw new Exception('Failed to create function environment: '.$stderr);
$orchestration->setCpus(App::getEnv('_APP_FUNCTIONS_CPUS', '1'));
$orchestration->setMemory(App::getEnv('_APP_FUNCTIONS_MEMORY', '256'));
$orchestration->setSwap(App::getEnv('_APP_FUNCTIONS_MEMORY_SWAP', '256'));
foreach($vars as &$value) {
$value = strval($value);
}
$exitCodeUntar = Console::execute("docker exec ".
$container.
" sh -c 'mv /tmp/code.tar.gz /usr/local/src/code.tar.gz && tar -zxf /usr/local/src/code.tar.gz --strip 1 && rm /usr/local/src/code.tar.gz'"
, '', $stdout, $stderr, 60);
$id = $orchestration->run(
image: $runtime['image'],
name: $container,
command: ['tail',
'-f',
'/dev/null'
],
entrypoint: '',
workdir: '/usr/local/src',
volumes: [],
vars: $vars,
mountFolder: $tagPathTargetDir,
labels: [
'appwrite-type' => 'function',
'appwrite-created' => strval($executionTime)
]);
if($exitCodeUntar !== 0) {
throw new Exception('Failed to extract tar: '.$stderr);
$untarStdout = '';
$untarStderr = '';
$untarSuccess = $orchestration->execute(
name: $container,
command: [
'sh',
'-c',
'mv /tmp/code.tar.gz /usr/local/src/code.tar.gz && tar -zxf /usr/local/src/code.tar.gz --strip 1 && rm /usr/local/src/code.tar.gz'
],
stdout: $untarStdout,
stderr: $untarStderr,
vars: $vars,
timeout: 60);
if (!$untarSuccess) {
throw new Exception('Failed to extract tar: '.$untarStderr);
}
$executionEnd = \microtime(true);
$list[$container] = [
'name' => $container,
'online' => true,
'status' => 'Up',
'labels' => [
$list[$container] = new Container($container, $id, 'Up',
[
'appwrite-type' => 'function',
'appwrite-created' => $executionTime,
],
];
'appwrite-created' => strval($executionTime),
]);
Console::info("Function created in " . ($executionEnd - $executionStart) . " seconds with exit code {$exitCode}");
Console::info('Function created in ' . ($executionEnd - $executionStart) . ' seconds');
}
else {
Console::info('Container is ready to run');
}
$stdout = '';
$stderr = '';
$executionStart = \microtime(true);
$exitCode = Console::execute("docker exec ".\implode(" ", $vars)." {$container} {$command}"
, '', $stdout, $stderr, $function->getAttribute('timeout', (int) App::getEnv('_APP_FUNCTIONS_TIMEOUT', 900)));
$exitCode = 0;
try {
$exitCode = (int)!$orchestration->execute(
name: $container,
command: $orchestration->parseCommandString($command),
stdout: $stdout,
stderr: $stderr,
vars: $vars,
timeout: $function->getAttribute('timeout', (int) App::getEnv('_APP_FUNCTIONS_TIMEOUT', 900)));
} catch (TimeoutException $e) {
$exitCode = 124;
} catch (OrchestrationException $e) {
$stderr = $e->getMessage();
$exitCode = 1;
}
$executionEnd = \microtime(true);
$executionTime = ($executionEnd - $executionStart);
$functionStatus = ($exitCode === 0) ? 'completed' : 'failed';
Console::info("Function executed in " . ($executionEnd - $executionStart) . " seconds with exit code {$exitCode}");
Console::info('Function executed in ' . ($executionEnd - $executionStart) . ' seconds, status: ' . $functionStatus);
Authorization::disable();
$execution = $database->updateDocument(array_merge($execution->getArrayCopy(), [
'tagId' => $tag->getId(),
'status' => $functionStatus,
'exitCode' => $exitCode,
'stdout' => \mb_substr($stdout, -4000), // log last 4000 chars output
'stderr' => \mb_substr($stderr, -4000), // log last 4000 chars output
'time' => $executionTime,
'time' => $executionTime
]));
Authorization::reset();
@ -529,6 +519,7 @@ class FunctionsV1 extends Worker
public function cleanup(): void
{
global $list;
global $orchestration;
Console::success(count($list).' running containers counted');
@ -538,19 +529,17 @@ class FunctionsV1 extends Worker
Console::info('Starting containers cleanup');
\uasort($list, function ($item1, $item2) {
return (int)($item1['appwrite-created'] ?? 0) <=> (int)($item2['appwrite-created'] ?? 0);
return (int)($item1->getLabels['appwrite-created'] ?? 0) <=> (int)($item2->getLabels['appwrite-created'] ?? 0);
});
while(\count($list) > $max) {
$first = \array_shift($list);
$stdout = '';
$stderr = '';
if(Console::execute("docker rm -f {$first['name']}", '', $stdout, $stderr, 30) !== 0) {
Console::error('Failed to remove container: '.$stderr);
}
else {
Console::info('Removed container: '.$first['name']);
try {
$orchestration->remove($first->getName(), true);
Console::info('Removed container: '.$first->getName());
} catch (Exception $e) {
Console::error('Failed to remove container: '.$e);
}
}
}

View file

@ -54,6 +54,7 @@
"utopia-php/image": "0.5.*",
"resque/php-resque": "1.3.6",
"matomo/device-detector": "4.2.3",
"utopia-php/orchestration": "0.2.0",
"dragonmantank/cron-expression": "3.1.0",
"influxdb/influxdb-php": "1.15.2",
"phpmailer/phpmailer": "6.5.0",

95
composer.lock generated
View file

@ -4,7 +4,7 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
"This file is @generated automatically"
],
"content-hash": "45963af754680568d89330a4f37c40d1",
"content-hash": "0a782184d016e458ff18f20a2c49ccfb",
"packages": [
{
"name": "adhocore/jwt",
@ -1907,6 +1907,61 @@
},
"time": "2021-07-24T11:35:55+00:00"
},
{
"name": "utopia-php/orchestration",
"version": "0.2.0",
"source": {
"type": "git",
"url": "https://github.com/utopia-php/orchestration.git",
"reference": "de10509017768cf2b62363bb39912002ab41dafb"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/utopia-php/orchestration/zipball/de10509017768cf2b62363bb39912002ab41dafb",
"reference": "de10509017768cf2b62363bb39912002ab41dafb",
"shasum": ""
},
"require": {
"php": ">=8.0",
"utopia-php/cli": "0.11.*"
},
"require-dev": {
"phpunit/phpunit": "^9.3",
"vimeo/psalm": "4.0.1"
},
"type": "library",
"autoload": {
"psr-4": {
"Utopia\\Orchestration\\": "src/Orchestration"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Eldad Fux",
"email": "eldad@appwrite.io"
}
],
"description": "Lite & fast micro PHP abstraction library for container orchestration",
"keywords": [
"docker",
"framework",
"kubernetes",
"orchestration",
"php",
"swarm",
"upf",
"utopia"
],
"support": {
"issues": "https://github.com/utopia-php/orchestration/issues",
"source": "https://github.com/utopia-php/orchestration/tree/0.2.0"
},
"time": "2021-08-16T12:52:42+00:00"
},
{
"name": "utopia-php/preloader",
"version": "0.2.4",
@ -2452,16 +2507,16 @@
},
{
"name": "composer/package-versions-deprecated",
"version": "1.11.99.2",
"version": "1.11.99.3",
"source": {
"type": "git",
"url": "https://github.com/composer/package-versions-deprecated.git",
"reference": "c6522afe5540d5fc46675043d3ed5a45a740b27c"
"reference": "fff576ac850c045158a250e7e27666e146e78d18"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/composer/package-versions-deprecated/zipball/c6522afe5540d5fc46675043d3ed5a45a740b27c",
"reference": "c6522afe5540d5fc46675043d3ed5a45a740b27c",
"url": "https://api.github.com/repos/composer/package-versions-deprecated/zipball/fff576ac850c045158a250e7e27666e146e78d18",
"reference": "fff576ac850c045158a250e7e27666e146e78d18",
"shasum": ""
},
"require": {
@ -2505,7 +2560,7 @@
"description": "Composer plugin that provides efficient querying for installed package versions (no runtime IO)",
"support": {
"issues": "https://github.com/composer/package-versions-deprecated/issues",
"source": "https://github.com/composer/package-versions-deprecated/tree/1.11.99.2"
"source": "https://github.com/composer/package-versions-deprecated/tree/1.11.99.3"
},
"funding": [
{
@ -2521,7 +2576,7 @@
"type": "tidelift"
}
],
"time": "2021-05-24T07:46:03+00:00"
"time": "2021-08-17T13:49:14+00:00"
},
{
"name": "composer/semver",
@ -5047,16 +5102,16 @@
},
{
"name": "symfony/console",
"version": "v5.3.6",
"version": "v5.3.7",
"source": {
"type": "git",
"url": "https://github.com/symfony/console.git",
"reference": "51b71afd6d2dc8f5063199357b9880cea8d8bfe2"
"reference": "8b1008344647462ae6ec57559da166c2bfa5e16a"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/symfony/console/zipball/51b71afd6d2dc8f5063199357b9880cea8d8bfe2",
"reference": "51b71afd6d2dc8f5063199357b9880cea8d8bfe2",
"url": "https://api.github.com/repos/symfony/console/zipball/8b1008344647462ae6ec57559da166c2bfa5e16a",
"reference": "8b1008344647462ae6ec57559da166c2bfa5e16a",
"shasum": ""
},
"require": {
@ -5126,7 +5181,7 @@
"terminal"
],
"support": {
"source": "https://github.com/symfony/console/tree/v5.3.6"
"source": "https://github.com/symfony/console/tree/v5.3.7"
},
"funding": [
{
@ -5142,7 +5197,7 @@
"type": "tidelift"
}
],
"time": "2021-07-27T19:10:22+00:00"
"time": "2021-08-25T20:02:16+00:00"
},
{
"name": "symfony/deprecation-contracts",
@ -5699,16 +5754,16 @@
},
{
"name": "symfony/string",
"version": "v5.3.3",
"version": "v5.3.7",
"source": {
"type": "git",
"url": "https://github.com/symfony/string.git",
"reference": "bd53358e3eccec6a670b5f33ab680d8dbe1d4ae1"
"reference": "8d224396e28d30f81969f083a58763b8b9ceb0a5"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/symfony/string/zipball/bd53358e3eccec6a670b5f33ab680d8dbe1d4ae1",
"reference": "bd53358e3eccec6a670b5f33ab680d8dbe1d4ae1",
"url": "https://api.github.com/repos/symfony/string/zipball/8d224396e28d30f81969f083a58763b8b9ceb0a5",
"reference": "8d224396e28d30f81969f083a58763b8b9ceb0a5",
"shasum": ""
},
"require": {
@ -5762,7 +5817,7 @@
"utf8"
],
"support": {
"source": "https://github.com/symfony/string/tree/v5.3.3"
"source": "https://github.com/symfony/string/tree/v5.3.7"
},
"funding": [
{
@ -5778,7 +5833,7 @@
"type": "tidelift"
}
],
"time": "2021-06-27T11:44:38+00:00"
"time": "2021-08-26T08:00:08+00:00"
},
{
"name": "theseer/tokenizer",
@ -6089,5 +6144,5 @@
"platform-overrides": {
"php": "8.0"
},
"plugin-api-version": "2.1.0"
"plugin-api-version": "2.0.0"
}

View file

@ -196,7 +196,6 @@ class FunctionsCustomClientTest extends Scope
]);
$output = json_decode($executions['body']['stdout'], true);
$this->assertEquals(200, $executions['headers']['status-code']);
$this->assertEquals('completed', $executions['body']['status']);
$this->assertEquals($functionId, $output['APPWRITE_FUNCTION_ID']);

View file

@ -518,7 +518,7 @@ class FunctionsCustomServerTest extends Scope
$this->assertEquals($executions['body']['executions'][0]['$id'], $executionId);
$this->assertEquals($executions['body']['executions'][0]['trigger'], 'http');
$this->assertEquals($executions['body']['executions'][0]['status'], 'failed');
$this->assertEquals($executions['body']['executions'][0]['exitCode'], 1);
$this->assertEquals($executions['body']['executions'][0]['exitCode'], 124);
$this->assertGreaterThan(2, $executions['body']['executions'][0]['time']);
$this->assertLessThan(3, $executions['body']['executions'][0]['time']);
$this->assertEquals($executions['body']['executions'][0]['stdout'], '');