diff --git a/CHANGES.md b/CHANGES.md index c63d0205e..11043925a 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,9 +1,15 @@ +# Version 0.10.0 (Not Released Yet) +- Switch from using Docker CLI to Docker API by intergrating [utopia-php/orchestration](https://github.com/utopia-php/orchestration) +- Added DOCKERHUB_PULL_USERNAME, DOCKERHUB_PULL_PASSWORD and DOCKERHUB_PULL_EMAIL env variables for pulling from + private DockerHub repos + # Version 0.9.4 ## Security - Fixed security vulnerability that exposes project ID's from other admin users (#1453) + # Version 0.9.3 ## Bugs diff --git a/app/config/variables.php b/app/config/variables.php index df7346cd5..bb505fd6b 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -462,6 +462,33 @@ return [ 'question' => '', 'filter' => '' ], + [ + 'name' => 'DOCKERHUB_PULL_USERNAME', + 'description' => 'The username for hub.docker.com. This variable is used to pull images from hub.docker.com.', + 'introduction' => '0.10.0', + 'default' => '', + 'required' => false, + 'question' => '', + 'filter' => '' + ], + [ + 'name' => 'DOCKERHUB_PULL_PASSWORD', + 'description' => 'The password for hub.docker.com. This variable is used to pull images from hub.docker.com.', + 'introduction' => '0.10.0', + 'default' => '', + 'required' => false, + 'question' => '', + 'filter' => '' + ], + [ + 'name' => 'DOCKERHUB_PULL_EMAIL', + 'description' => 'The email for hub.docker.com. This variable is used to pull images from hub.docker.com.', + 'introduction' => '0.10.0', + 'default' => '', + 'required' => false, + 'question' => '', + 'filter' => '' + ], ], [ 'category' => 'Maintenance', diff --git a/app/workers/functions.php b/app/workers/functions.php index 2ab1f1f30..55387422e 100644 --- a/app/workers/functions.php +++ b/app/workers/functions.php @@ -13,6 +13,11 @@ use Swoole\Runtime; use Utopia\App; use Utopia\CLI\Console; use Utopia\Config\Config; +use Utopia\Orchestration\Orchestration; +use Utopia\Orchestration\Adapter\DockerAPI; +use Utopia\Orchestration\Container; +use Utopia\Orchestration\Exception\Orchestration as OrchestrationException; +use Utopia\Orchestration\Exception\Timeout as TimeoutException; require_once __DIR__.'/../workers.php'; @@ -23,39 +28,27 @@ Console::success(APP_NAME.' functions worker v1 has started'); $runtimes = Config::getParam('runtimes'); +$dockerUser = App::getEnv('DOCKERHUB_PULL_USERNAME', null); +$dockerPass = App::getEnv('DOCKERHUB_PULL_PASSWORD', null); +$dockerEmail = App::getEnv('DOCKERHUB_PULL_EMAIL', null); +$orchestration = new Orchestration(new DockerAPI($dockerUser, $dockerPass, $dockerEmail)); + /** * Warmup Docker Images */ $warmupStart = \microtime(true); -Co\run(function() use ($runtimes) { // Warmup: make sure images are ready to run fast 🚀 - - $dockerUser = App::getEnv('DOCKERHUB_PULL_USERNAME', null); - $dockerPass = App::getEnv('DOCKERHUB_PULL_PASSWORD', null); - - if($dockerUser) { - $stdout = ''; - $stderr = ''; - - Console::execute('docker login --username '.$dockerUser.' --password-stdin', $dockerPass, $stdout, $stderr); - Console::log('Docker Login'. $stdout.$stderr); - } - +Co\run(function() use ($runtimes, $orchestration) { // Warmup: make sure images are ready to run fast 🚀 foreach($runtimes as $runtime) { - go(function() use ($runtime) { - $stdout = ''; - $stderr = ''; - + go(function() use ($runtime, $orchestration) { Console::info('Warming up '.$runtime['name'].' '.$runtime['version'].' environment...'); - - Console::execute('docker pull '.$runtime['image'], '', $stdout, $stderr); - - if(!empty($stdout)) { - Console::log($stdout); - } - - if(!empty($stderr)) { - Console::error($stderr); + + $response = $orchestration->pull($runtime['image']); + + if ($response) { + Console::success("Successfully Warmed up {$runtime['name']} {$runtime['version']}!"); + } else { + Console::error("Failed to Warmup {$runtime['name']} {$runtime['version']}!"); } }); } @@ -74,40 +67,17 @@ $stderr = ''; $executionStart = \microtime(true); -$exitCode = Console::execute('docker ps --all --format "name={{.Names}}&status={{.Status}}&labels={{.Labels}}" --filter label=appwrite-type=function' - , '', $stdout, $stderr, 30); +$response = $orchestration->list(['label' => 'appwrite-type=function']); + +$list = []; + +foreach ($response as $value) { + $list[$value->getName()] = $value; +} $executionEnd = \microtime(true); -$list = []; -$stdout = \explode("\n", $stdout); - -\array_map(function($value) use (&$list) { - $container = []; - - \parse_str($value, $container); - - if(isset($container['name'])) { - $container = [ - 'name' => $container['name'], - 'online' => (\substr($container['status'], 0, 2) === 'Up'), - 'status' => $container['status'], - 'labels' => $container['labels'], - ]; - - \array_map(function($value) use (&$container) { - $value = \explode('=', $value); - - if(isset($value[0]) && isset($value[1])) { - $container[$value[0]] = $value[1]; - } - }, \explode(',', $container['labels'])); - - $list[$container['name']] = $container; - } -}, $stdout); - -Console::info(count($list)." functions listed in " . ($executionEnd - $executionStart) . " seconds with exit code {$exitCode}"); +Console::info(count($list).' functions listed in ' . ($executionEnd - $executionStart) . ' seconds'); /** * 1. Get event args - DONE @@ -297,6 +267,7 @@ class FunctionsV1 extends Worker public function execute(string $trigger, string $projectId, string $executionId, Database $database, Document $function, string $event = '', string $eventData = '', string $data = '', array $webhooks = [], string $userId = '', string $jwt = ''): void { global $list; + global $orchestration; $runtimes = Config::getParam('runtimes'); @@ -355,12 +326,6 @@ class FunctionsV1 extends Worker 'APPWRITE_FUNCTION_PROJECT_ID' => $projectId, ]); - \array_walk($vars, function (&$value, $key) { - $key = $this->filterEnvKey($key); - $value = \escapeshellarg((empty($value)) ? '' : $value); - $value = "--env {$key}={$value}"; - }); - $tagPath = $tag->getAttribute('path', ''); $tagPathTarget = '/tmp/project-'.$projectId.'/'.$tag->getId().'/code.tar.gz'; $tagPathTargetDir = \pathinfo($tagPathTarget, PATHINFO_DIRNAME); @@ -383,12 +348,14 @@ class FunctionsV1 extends Worker } } - if(isset($list[$container]) && !$list[$container]['online']) { // Remove conatiner if not online + if(isset($list[$container]) && !(\substr($list[$container]->getStatus(), 0, 2) === 'Up')) { // Remove conatiner if not online $stdout = ''; $stderr = ''; - - if(Console::execute("docker rm {$container}", '', $stdout, $stderr, 30) !== 0) { - throw new Exception('Failed to remove offline container: '.$stderr); + + try { + $orchestration->remove($container); + } catch (Exception $e) { + Console::warning('Failed to remove container: '.$e->getMessage()); } unset($list[$container]); @@ -409,79 +376,102 @@ class FunctionsV1 extends Worker $executionStart = \microtime(true); $executionTime = \time(); - $cpus = App::getEnv('_APP_FUNCTIONS_CPUS', ''); - $memory = App::getEnv('_APP_FUNCTIONS_MEMORY', ''); - $swap = App::getEnv('_APP_FUNCTIONS_MEMORY_SWAP', ''); - $exitCode = Console::execute("docker run ". - " -d". - " --entrypoint=\"\"". - (empty($cpus) ? "" : (" --cpus=".$cpus)). - (empty($memory) ? "" : (" --memory=".$memory."m")). - (empty($swap) ? "" : (" --memory-swap=".$swap."m")). - " --name={$container}". - " --label appwrite-type=function". - " --label appwrite-created={$executionTime}". - " --volume {$tagPathTargetDir}:/tmp:rw". - " --workdir /usr/local/src". - " ".\implode(" ", $vars). - " {$runtime['image']}". - " tail -f /dev/null" - , '', $stdout, $stderr, 30); - if($exitCode !== 0) { - throw new Exception('Failed to create function environment: '.$stderr); + $orchestration->setCpus(App::getEnv('_APP_FUNCTIONS_CPUS', '1')); + $orchestration->setMemory(App::getEnv('_APP_FUNCTIONS_MEMORY', '256')); + $orchestration->setSwap(App::getEnv('_APP_FUNCTIONS_MEMORY_SWAP', '256')); + + foreach($vars as &$value) { + $value = strval($value); } - $exitCodeUntar = Console::execute("docker exec ". - $container. - " sh -c 'mv /tmp/code.tar.gz /usr/local/src/code.tar.gz && tar -zxf /usr/local/src/code.tar.gz --strip 1 && rm /usr/local/src/code.tar.gz'" - , '', $stdout, $stderr, 60); + $id = $orchestration->run( + image: $runtime['image'], + name: $container, + command: ['tail', + '-f', + '/dev/null' + ], + entrypoint: '', + workdir: '/usr/local/src', + volumes: [], + vars: $vars, + mountFolder: $tagPathTargetDir, + labels: [ + 'appwrite-type' => 'function', + 'appwrite-created' => strval($executionTime) + ]); - if($exitCodeUntar !== 0) { - throw new Exception('Failed to extract tar: '.$stderr); + $untarStdout = ''; + $untarStderr = ''; + + $untarSuccess = $orchestration->execute( + name: $container, + command: [ + 'sh', + '-c', + 'mv /tmp/code.tar.gz /usr/local/src/code.tar.gz && tar -zxf /usr/local/src/code.tar.gz --strip 1 && rm /usr/local/src/code.tar.gz' + ], + stdout: $untarStdout, + stderr: $untarStderr, + vars: $vars, + timeout: 60); + + if (!$untarSuccess) { + throw new Exception('Failed to extract tar: '.$untarStderr); } $executionEnd = \microtime(true); - $list[$container] = [ - 'name' => $container, - 'online' => true, - 'status' => 'Up', - 'labels' => [ + $list[$container] = new Container($container, $id, 'Up', + [ 'appwrite-type' => 'function', - 'appwrite-created' => $executionTime, - ], - ]; + 'appwrite-created' => strval($executionTime), + ]); - Console::info("Function created in " . ($executionEnd - $executionStart) . " seconds with exit code {$exitCode}"); + Console::info('Function created in ' . ($executionEnd - $executionStart) . ' seconds'); } else { Console::info('Container is ready to run'); } - + $stdout = ''; $stderr = ''; $executionStart = \microtime(true); - - $exitCode = Console::execute("docker exec ".\implode(" ", $vars)." {$container} {$command}" - , '', $stdout, $stderr, $function->getAttribute('timeout', (int) App::getEnv('_APP_FUNCTIONS_TIMEOUT', 900))); + + $exitCode = 0; + + try { + $exitCode = (int)!$orchestration->execute( + name: $container, + command: $orchestration->parseCommandString($command), + stdout: $stdout, + stderr: $stderr, + vars: $vars, + timeout: $function->getAttribute('timeout', (int) App::getEnv('_APP_FUNCTIONS_TIMEOUT', 900))); + } catch (TimeoutException $e) { + $exitCode = 124; + } catch (OrchestrationException $e) { + $stderr = $e->getMessage(); + $exitCode = 1; + } $executionEnd = \microtime(true); $executionTime = ($executionEnd - $executionStart); $functionStatus = ($exitCode === 0) ? 'completed' : 'failed'; - Console::info("Function executed in " . ($executionEnd - $executionStart) . " seconds with exit code {$exitCode}"); + Console::info('Function executed in ' . ($executionEnd - $executionStart) . ' seconds, status: ' . $functionStatus); Authorization::disable(); - + $execution = $database->updateDocument(array_merge($execution->getArrayCopy(), [ 'tagId' => $tag->getId(), 'status' => $functionStatus, 'exitCode' => $exitCode, 'stdout' => \mb_substr($stdout, -4000), // log last 4000 chars output 'stderr' => \mb_substr($stderr, -4000), // log last 4000 chars output - 'time' => $executionTime, + 'time' => $executionTime ])); Authorization::reset(); @@ -529,6 +519,7 @@ class FunctionsV1 extends Worker public function cleanup(): void { global $list; + global $orchestration; Console::success(count($list).' running containers counted'); @@ -538,19 +529,17 @@ class FunctionsV1 extends Worker Console::info('Starting containers cleanup'); \uasort($list, function ($item1, $item2) { - return (int)($item1['appwrite-created'] ?? 0) <=> (int)($item2['appwrite-created'] ?? 0); + return (int)($item1->getLabels['appwrite-created'] ?? 0) <=> (int)($item2->getLabels['appwrite-created'] ?? 0); }); while(\count($list) > $max) { $first = \array_shift($list); - $stdout = ''; - $stderr = ''; - if(Console::execute("docker rm -f {$first['name']}", '', $stdout, $stderr, 30) !== 0) { - Console::error('Failed to remove container: '.$stderr); - } - else { - Console::info('Removed container: '.$first['name']); + try { + $orchestration->remove($first->getName(), true); + Console::info('Removed container: '.$first->getName()); + } catch (Exception $e) { + Console::error('Failed to remove container: '.$e); } } } diff --git a/composer.json b/composer.json index 9c1807312..94aad9771 100644 --- a/composer.json +++ b/composer.json @@ -54,6 +54,7 @@ "utopia-php/image": "0.5.*", "resque/php-resque": "1.3.6", "matomo/device-detector": "4.2.3", + "utopia-php/orchestration": "0.2.0", "dragonmantank/cron-expression": "3.1.0", "influxdb/influxdb-php": "1.15.2", "phpmailer/phpmailer": "6.5.0", diff --git a/composer.lock b/composer.lock index e0858de93..d1fa6eecf 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "45963af754680568d89330a4f37c40d1", + "content-hash": "0a782184d016e458ff18f20a2c49ccfb", "packages": [ { "name": "adhocore/jwt", @@ -1907,6 +1907,61 @@ }, "time": "2021-07-24T11:35:55+00:00" }, + { + "name": "utopia-php/orchestration", + "version": "0.2.0", + "source": { + "type": "git", + "url": "https://github.com/utopia-php/orchestration.git", + "reference": "de10509017768cf2b62363bb39912002ab41dafb" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/utopia-php/orchestration/zipball/de10509017768cf2b62363bb39912002ab41dafb", + "reference": "de10509017768cf2b62363bb39912002ab41dafb", + "shasum": "" + }, + "require": { + "php": ">=8.0", + "utopia-php/cli": "0.11.*" + }, + "require-dev": { + "phpunit/phpunit": "^9.3", + "vimeo/psalm": "4.0.1" + }, + "type": "library", + "autoload": { + "psr-4": { + "Utopia\\Orchestration\\": "src/Orchestration" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Eldad Fux", + "email": "eldad@appwrite.io" + } + ], + "description": "Lite & fast micro PHP abstraction library for container orchestration", + "keywords": [ + "docker", + "framework", + "kubernetes", + "orchestration", + "php", + "swarm", + "upf", + "utopia" + ], + "support": { + "issues": "https://github.com/utopia-php/orchestration/issues", + "source": "https://github.com/utopia-php/orchestration/tree/0.2.0" + }, + "time": "2021-08-16T12:52:42+00:00" + }, { "name": "utopia-php/preloader", "version": "0.2.4", @@ -2452,16 +2507,16 @@ }, { "name": "composer/package-versions-deprecated", - "version": "1.11.99.2", + "version": "1.11.99.3", "source": { "type": "git", "url": "https://github.com/composer/package-versions-deprecated.git", - "reference": "c6522afe5540d5fc46675043d3ed5a45a740b27c" + "reference": "fff576ac850c045158a250e7e27666e146e78d18" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/composer/package-versions-deprecated/zipball/c6522afe5540d5fc46675043d3ed5a45a740b27c", - "reference": "c6522afe5540d5fc46675043d3ed5a45a740b27c", + "url": "https://api.github.com/repos/composer/package-versions-deprecated/zipball/fff576ac850c045158a250e7e27666e146e78d18", + "reference": "fff576ac850c045158a250e7e27666e146e78d18", "shasum": "" }, "require": { @@ -2505,7 +2560,7 @@ "description": "Composer plugin that provides efficient querying for installed package versions (no runtime IO)", "support": { "issues": "https://github.com/composer/package-versions-deprecated/issues", - "source": "https://github.com/composer/package-versions-deprecated/tree/1.11.99.2" + "source": "https://github.com/composer/package-versions-deprecated/tree/1.11.99.3" }, "funding": [ { @@ -2521,7 +2576,7 @@ "type": "tidelift" } ], - "time": "2021-05-24T07:46:03+00:00" + "time": "2021-08-17T13:49:14+00:00" }, { "name": "composer/semver", @@ -5047,16 +5102,16 @@ }, { "name": "symfony/console", - "version": "v5.3.6", + "version": "v5.3.7", "source": { "type": "git", "url": "https://github.com/symfony/console.git", - "reference": "51b71afd6d2dc8f5063199357b9880cea8d8bfe2" + "reference": "8b1008344647462ae6ec57559da166c2bfa5e16a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/console/zipball/51b71afd6d2dc8f5063199357b9880cea8d8bfe2", - "reference": "51b71afd6d2dc8f5063199357b9880cea8d8bfe2", + "url": "https://api.github.com/repos/symfony/console/zipball/8b1008344647462ae6ec57559da166c2bfa5e16a", + "reference": "8b1008344647462ae6ec57559da166c2bfa5e16a", "shasum": "" }, "require": { @@ -5126,7 +5181,7 @@ "terminal" ], "support": { - "source": "https://github.com/symfony/console/tree/v5.3.6" + "source": "https://github.com/symfony/console/tree/v5.3.7" }, "funding": [ { @@ -5142,7 +5197,7 @@ "type": "tidelift" } ], - "time": "2021-07-27T19:10:22+00:00" + "time": "2021-08-25T20:02:16+00:00" }, { "name": "symfony/deprecation-contracts", @@ -5699,16 +5754,16 @@ }, { "name": "symfony/string", - "version": "v5.3.3", + "version": "v5.3.7", "source": { "type": "git", "url": "https://github.com/symfony/string.git", - "reference": "bd53358e3eccec6a670b5f33ab680d8dbe1d4ae1" + "reference": "8d224396e28d30f81969f083a58763b8b9ceb0a5" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/string/zipball/bd53358e3eccec6a670b5f33ab680d8dbe1d4ae1", - "reference": "bd53358e3eccec6a670b5f33ab680d8dbe1d4ae1", + "url": "https://api.github.com/repos/symfony/string/zipball/8d224396e28d30f81969f083a58763b8b9ceb0a5", + "reference": "8d224396e28d30f81969f083a58763b8b9ceb0a5", "shasum": "" }, "require": { @@ -5762,7 +5817,7 @@ "utf8" ], "support": { - "source": "https://github.com/symfony/string/tree/v5.3.3" + "source": "https://github.com/symfony/string/tree/v5.3.7" }, "funding": [ { @@ -5778,7 +5833,7 @@ "type": "tidelift" } ], - "time": "2021-06-27T11:44:38+00:00" + "time": "2021-08-26T08:00:08+00:00" }, { "name": "theseer/tokenizer", @@ -6089,5 +6144,5 @@ "platform-overrides": { "php": "8.0" }, - "plugin-api-version": "2.1.0" + "plugin-api-version": "2.0.0" } diff --git a/tests/e2e/Services/Functions/FunctionsCustomClientTest.php b/tests/e2e/Services/Functions/FunctionsCustomClientTest.php index 454a149c3..4b52aa26c 100644 --- a/tests/e2e/Services/Functions/FunctionsCustomClientTest.php +++ b/tests/e2e/Services/Functions/FunctionsCustomClientTest.php @@ -196,7 +196,6 @@ class FunctionsCustomClientTest extends Scope ]); $output = json_decode($executions['body']['stdout'], true); - $this->assertEquals(200, $executions['headers']['status-code']); $this->assertEquals('completed', $executions['body']['status']); $this->assertEquals($functionId, $output['APPWRITE_FUNCTION_ID']); diff --git a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php index 775eca06e..58740150d 100644 --- a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php +++ b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php @@ -518,7 +518,7 @@ class FunctionsCustomServerTest extends Scope $this->assertEquals($executions['body']['executions'][0]['$id'], $executionId); $this->assertEquals($executions['body']['executions'][0]['trigger'], 'http'); $this->assertEquals($executions['body']['executions'][0]['status'], 'failed'); - $this->assertEquals($executions['body']['executions'][0]['exitCode'], 1); + $this->assertEquals($executions['body']['executions'][0]['exitCode'], 124); $this->assertGreaterThan(2, $executions['body']['executions'][0]['time']); $this->assertLessThan(3, $executions['body']['executions'][0]['time']); $this->assertEquals($executions['body']['executions'][0]['stdout'], '');