From a9e7b617c050f029ed5aae7d45a4cee00515aee4 Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Thu, 22 Jul 2021 13:49:52 +0100 Subject: [PATCH 01/20] First Iteration --- app/workers/functions.php | 197 ++++++++++++++++---------------------- composer.json | 7 ++ 2 files changed, 92 insertions(+), 112 deletions(-) diff --git a/app/workers/functions.php b/app/workers/functions.php index a85843b38..ddac808c7 100644 --- a/app/workers/functions.php +++ b/app/workers/functions.php @@ -13,6 +13,10 @@ use Swoole\Runtime; use Utopia\App; use Utopia\CLI\Console; use Utopia\Config\Config; +use Utopia\Orchestration\Orchestration; +use Utopia\Orchestration\Adapter\DockerAPI; +use Utopia\Orchestration\Container; +use Utopia\Orchestration\Exceptions\TimeoutException; require_once __DIR__.'/../workers.php'; @@ -23,39 +27,24 @@ Console::success(APP_NAME.' functions worker v1 has started'); $runtimes = Config::getParam('runtimes'); +$dockerUser = App::getEnv('DOCKERHUB_PULL_USERNAME', null); +$dockerPass = App::getEnv('DOCKERHUB_PULL_PASSWORD', null); +$orchestration = new Orchestration(new DockerAPI($dockerUser, $dockerPass)); + /** * Warmup Docker Images */ $warmupStart = \microtime(true); -Co\run(function() use ($runtimes) { // Warmup: make sure images are ready to run fast 🚀 - - $dockerUser = App::getEnv('DOCKERHUB_PULL_USERNAME', null); - $dockerPass = App::getEnv('DOCKERHUB_PULL_PASSWORD', null); - - if($dockerUser) { - $stdout = ''; - $stderr = ''; - - Console::execute('docker login --username '.$dockerUser.' --password-stdin', $dockerPass, $stdout, $stderr); - Console::log('Docker Login'. $stdout.$stderr); - } - +Co\run(function() use ($runtimes, $orchestration) { // Warmup: make sure images are ready to run fast 🚀 foreach($runtimes as $runtime) { - go(function() use ($runtime) { - $stdout = ''; - $stderr = ''; - + go(function() use ($runtime, $orchestration) { Console::info('Warming up '.$runtime['name'].' '.$runtime['version'].' environment...'); - - Console::execute('docker pull '.$runtime['image'], '', $stdout, $stderr); - - if(!empty($stdout)) { - Console::log($stdout); - } - - if(!empty($stderr)) { - Console::error($stderr); + try { + $orchestration->pull($runtime['image']); + Console::success("Successfully Warmed up {$runtime['name']} {$runtime['version']}!"); + } catch (Exception $e) { + Console::error($e); } }); } @@ -74,40 +63,21 @@ $stderr = ''; $executionStart = \microtime(true); -$exitCode = Console::execute('docker ps --all --format "name={{.Names}}&status={{.Status}}&labels={{.Labels}}" --filter label=appwrite-type=function' - , '', $stdout, $stderr, 30); +$response = $orchestration->list(); + +$list = []; + +array_map(function($value) use (&$list) { + if (!empty($value->labels["appwrite-type"])) { + if ($value->labels["appwrite-type"] == "function") { + $list[$value->name] = $value; + } + } +}, $response); $executionEnd = \microtime(true); -$list = []; -$stdout = \explode("\n", $stdout); - -\array_map(function($value) use (&$list) { - $container = []; - - \parse_str($value, $container); - - if(isset($container['name'])) { - $container = [ - 'name' => $container['name'], - 'online' => (\substr($container['status'], 0, 2) === 'Up'), - 'status' => $container['status'], - 'labels' => $container['labels'], - ]; - - \array_map(function($value) use (&$container) { - $value = \explode('=', $value); - - if(isset($value[0]) && isset($value[1])) { - $container[$value[0]] = $value[1]; - } - }, \explode(',', $container['labels'])); - - $list[$container['name']] = $container; - } -}, $stdout); - -Console::info(count($list)." functions listed in " . ($executionEnd - $executionStart) . " seconds with exit code {$exitCode}"); +Console::info(count($list)." functions listed in " . ($executionEnd - $executionStart) . " seconds"); /** * 1. Get event args - DONE @@ -297,6 +267,7 @@ class FunctionsV1 extends Worker public function execute(string $trigger, string $projectId, string $executionId, Database $database, Document $function, string $event = '', string $eventData = '', string $data = '', array $webhooks = [], string $userId = '', string $jwt = ''): void { global $list; + global $orchestration; $runtimes = Config::getParam('runtimes'); @@ -355,12 +326,6 @@ class FunctionsV1 extends Worker 'APPWRITE_FUNCTION_PROJECT_ID' => $projectId, ]); - \array_walk($vars, function (&$value, $key) { - $key = $this->filterEnvKey($key); - $value = \escapeshellarg((empty($value)) ? '' : $value); - $value = "--env {$key}={$value}"; - }); - $tagPath = $tag->getAttribute('path', ''); $tagPathTarget = '/tmp/project-'.$projectId.'/'.$tag->getId().'/code.tar.gz'; $tagPathTargetDir = \pathinfo($tagPathTarget, PATHINFO_DIRNAME); @@ -383,13 +348,11 @@ class FunctionsV1 extends Worker } } - if(isset($list[$container]) && !$list[$container]['online']) { // Remove conatiner if not online + if(isset($list[$container]) && !(\substr($list[$container]->status, 0, 2) === 'Up')) { // Remove conatiner if not online $stdout = ''; $stderr = ''; - - if(Console::execute("docker rm {$container}", '', $stdout, $stderr, 30) !== 0) { - throw new Exception('Failed to remove offline container: '.$stderr); - } + + $orchestration->remove($container); unset($list[$container]); } @@ -412,39 +375,45 @@ class FunctionsV1 extends Worker $cpus = App::getEnv('_APP_FUNCTIONS_CPUS', ''); $memory = App::getEnv('_APP_FUNCTIONS_MEMORY', ''); $swap = App::getEnv('_APP_FUNCTIONS_MEMORY_SWAP', ''); - $exitCode = Console::execute("docker run ". - " -d". - " --entrypoint=\"\"". - (empty($cpus) ? "" : (" --cpus=".$cpus)). - (empty($memory) ? "" : (" --memory=".$memory."m")). - (empty($swap) ? "" : (" --memory-swap=".$swap."m")). - " --name={$container}". - " --label appwrite-type=function". - " --label appwrite-created={$executionTime}". - " --volume {$tagPathTargetDir}:/tmp:rw". - " --workdir /usr/local/src". - " ".\implode(" ", $vars). - " {$runtime['image']}". - " sh -c 'mv /tmp/code.tar.gz /usr/local/src/code.tar.gz && tar -zxf /usr/local/src/code.tar.gz --strip 1 && rm /usr/local/src/code.tar.gz && tail -f /dev/null'" - , '', $stdout, $stderr, 30); + + $orchestration->setCpus($cpus); + $orchestration->setMemory($memory); + $orchestration->setSwap($swap); + + \array_walk($vars, function (string &$value, string $key) { + $value = strval($value); + }); + + $response = $orchestration->run( + $runtime['image'], + $container, + "", + array( + 'sh', + '-c', + 'mv /tmp/code.tar.gz /usr/local/src/code.tar.gz && tar -zxf /usr/local/src/code.tar.gz --strip 1 && rm /usr/local/src/code.tar.gz && tail -f /dev/null' + ), + "/usr/local/src", + array(), + array(), + $tagPathTargetDir, + array( + "appwrite-type" => "function", + "appwrite-created" => "{$executionTime}" + )); $executionEnd = \microtime(true); - - if($exitCode !== 0) { - throw new Exception('Failed to create function environment: '.$stderr); - } - $list[$container] = [ - 'name' => $container, - 'online' => true, - 'status' => 'Up', - 'labels' => [ - 'appwrite-type' => 'function', - 'appwrite-created' => $executionTime, - ], - ]; + $list[$container] = new Container(); + + $list[$container]->name = $container; + $list[$container]->status = "Up"; + $list[$container]->labels = array( + 'appwrite-type' => 'function', + 'appwrite-created' => "{$executionTime}", + ); - Console::info("Function created in " . ($executionEnd - $executionStart) . " seconds with exit code {$exitCode}"); + Console::info("Function created in " . ($executionEnd - $executionStart) . " seconds"); } else { Console::info('Container is ready to run'); @@ -454,25 +423,30 @@ class FunctionsV1 extends Worker $stderr = ''; $executionStart = \microtime(true); - - $exitCode = Console::execute("docker exec ".\implode(" ", $vars)." {$container} {$command}" - , '', $stdout, $stderr, $function->getAttribute('timeout', (int) App::getEnv('_APP_FUNCTIONS_TIMEOUT', 900))); + + $exitCode = 0; + + try { + $exitCode = $orchestration->execute($container, $orchestration->parseCommandString($command), $stdout, $stderr, $vars, $function->getAttribute('timeout', (int) App::getEnv('_APP_FUNCTIONS_TIMEOUT', 900))); + } catch (TimeoutException $e) { + $exitCode = 0; + } $executionEnd = \microtime(true); $executionTime = ($executionEnd - $executionStart); - $functionStatus = ($exitCode === 0) ? 'completed' : 'failed'; + $functionStatus = $exitCode ? 'completed' : 'failed'; - Console::info("Function executed in " . ($executionEnd - $executionStart) . " seconds with exit code {$exitCode}"); + Console::info("Function executed in " . ($executionEnd - $executionStart) . " seconds, status: {$functionStatus} "); Authorization::disable(); $execution = $database->updateDocument(array_merge($execution->getArrayCopy(), [ 'tagId' => $tag->getId(), 'status' => $functionStatus, - 'exitCode' => $exitCode, + 'exitCode' => ($exitCode ? 0 : 1), 'stdout' => \mb_substr($stdout, -4000), // log last 4000 chars output 'stderr' => \mb_substr($stderr, -4000), // log last 4000 chars output - 'time' => $executionTime, + 'time' => $executionTime ])); Authorization::reset(); @@ -520,6 +494,7 @@ class FunctionsV1 extends Worker public function cleanup(): void { global $list; + global $orchestration; Console::success(count($list).' running containers counted'); @@ -529,19 +504,17 @@ class FunctionsV1 extends Worker Console::info('Starting containers cleanup'); \uasort($list, function ($item1, $item2) { - return (int)($item1['appwrite-created'] ?? 0) <=> (int)($item2['appwrite-created'] ?? 0); + return (int)($item1->labels['appwrite-created'] ?? 0) <=> (int)($item2->labels['appwrite-created'] ?? 0); }); while(\count($list) > $max) { $first = \array_shift($list); - $stdout = ''; - $stderr = ''; - if(Console::execute("docker rm -f {$first['name']}", '', $stdout, $stderr, 30) !== 0) { - Console::error('Failed to remove container: '.$stderr); - } - else { - Console::info('Removed container: '.$first['name']); + try { + $orchestration->remove($first->name, true); + Console::info('Removed container: '.$first->name); + } catch (Exception $e) { + Console::error('Failed to remove container: '.$e); } } } diff --git a/composer.json b/composer.json index 7aa0d4f01..671660055 100644 --- a/composer.json +++ b/composer.json @@ -9,6 +9,12 @@ "email": "eldad@appwrite.io" } ], + "repositories": [ + { + "type": "vcs", + "url": "https://github.com/PineappleIOnic/orchestration" + } + ], "autoload": { "psr-4": { "Appwrite\\": "src/Appwrite" @@ -54,6 +60,7 @@ "utopia-php/image": "0.5.*", "resque/php-resque": "1.3.6", "matomo/device-detector": "4.2.3", + "utopia-php/orchestration": "dev-dev", "dragonmantank/cron-expression": "3.1.0", "influxdb/influxdb-php": "1.15.2", "phpmailer/phpmailer": "6.5.0", From 4d2f5428a2f3e06f273e06cd5e6f3229f6c46cca Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Thu, 22 Jul 2021 14:53:30 +0100 Subject: [PATCH 02/20] Update to use protected container class --- app/workers/functions.php | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/app/workers/functions.php b/app/workers/functions.php index ddac808c7..cb6608069 100644 --- a/app/workers/functions.php +++ b/app/workers/functions.php @@ -384,7 +384,7 @@ class FunctionsV1 extends Worker $value = strval($value); }); - $response = $orchestration->run( + $id = $orchestration->run( $runtime['image'], $container, "", @@ -404,14 +404,13 @@ class FunctionsV1 extends Worker $executionEnd = \microtime(true); - $list[$container] = new Container(); - - $list[$container]->name = $container; - $list[$container]->status = "Up"; - $list[$container]->labels = array( - 'appwrite-type' => 'function', - 'appwrite-created' => "{$executionTime}", - ); + $list[$container] = new Container($container, + $id, + "Up", + array( + 'appwrite-type' => 'function', + 'appwrite-created' => "{$executionTime}", + )); Console::info("Function created in " . ($executionEnd - $executionStart) . " seconds"); } From e0ab558f9045a964719d0ca9dcee23827450113b Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Thu, 22 Jul 2021 14:54:23 +0100 Subject: [PATCH 03/20] Update functions.php --- app/workers/functions.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/workers/functions.php b/app/workers/functions.php index cb6608069..a0f2847c3 100644 --- a/app/workers/functions.php +++ b/app/workers/functions.php @@ -348,7 +348,7 @@ class FunctionsV1 extends Worker } } - if(isset($list[$container]) && !(\substr($list[$container]->status, 0, 2) === 'Up')) { // Remove conatiner if not online + if(isset($list[$container]) && !(\substr($list[$container]->getStatus(), 0, 2) === 'Up')) { // Remove conatiner if not online $stdout = ''; $stderr = ''; From b5ab777cadb68702b969a4dfee3e577c4be81aed Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Fri, 23 Jul 2021 09:48:51 +0100 Subject: [PATCH 04/20] Implement new native filters --- app/workers/functions.php | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/app/workers/functions.php b/app/workers/functions.php index a0f2847c3..18a70558e 100644 --- a/app/workers/functions.php +++ b/app/workers/functions.php @@ -63,16 +63,12 @@ $stderr = ''; $executionStart = \microtime(true); -$response = $orchestration->list(); +$response = $orchestration->list(['labels' => 'appwrite-type=function']); $list = []; array_map(function($value) use (&$list) { - if (!empty($value->labels["appwrite-type"])) { - if ($value->labels["appwrite-type"] == "function") { - $list[$value->name] = $value; - } - } + $list[$value->getName()] = $value; }, $response); $executionEnd = \microtime(true); From a4875cfd2297e44e2d159d3f5ac205f4687ec813 Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Fri, 23 Jul 2021 10:05:59 +0100 Subject: [PATCH 05/20] Fix Native Labels --- app/workers/functions.php | 8 +-- composer.lock | 107 ++++++++++++++++++++++++++++++-------- 2 files changed, 88 insertions(+), 27 deletions(-) diff --git a/app/workers/functions.php b/app/workers/functions.php index 18a70558e..227476add 100644 --- a/app/workers/functions.php +++ b/app/workers/functions.php @@ -63,7 +63,7 @@ $stderr = ''; $executionStart = \microtime(true); -$response = $orchestration->list(['labels' => 'appwrite-type=function']); +$response = $orchestration->list(['label' => 'appwrite-type=function']); $list = []; @@ -499,15 +499,15 @@ class FunctionsV1 extends Worker Console::info('Starting containers cleanup'); \uasort($list, function ($item1, $item2) { - return (int)($item1->labels['appwrite-created'] ?? 0) <=> (int)($item2->labels['appwrite-created'] ?? 0); + return (int)($item1->getLabels['appwrite-created'] ?? 0) <=> (int)($item2->getLabels['appwrite-created'] ?? 0); }); while(\count($list) > $max) { $first = \array_shift($list); try { - $orchestration->remove($first->name, true); - Console::info('Removed container: '.$first->name); + $orchestration->remove($first->getName(), true); + Console::info('Removed container: '.$first->getName()); } catch (Exception $e) { Console::error('Failed to remove container: '.$e); } diff --git a/composer.lock b/composer.lock index fba3b66ce..f6485ea18 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "e24aacead283e33130051470bb4312e6", + "content-hash": "e30c92bf02f5aa68c1fd67aa985a7949", "packages": [ { "name": "adhocore/jwt", @@ -1907,6 +1907,64 @@ }, "time": "2020-10-24T08:12:55+00:00" }, + { + "name": "utopia-php/orchestration", + "version": "dev-dev", + "source": { + "type": "git", + "url": "https://github.com/PineappleIOnic/orchestration.git", + "reference": "b691209cd68392553fe38a57bcb7c274a4c56ead" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/PineappleIOnic/orchestration/zipball/b691209cd68392553fe38a57bcb7c274a4c56ead", + "reference": "b691209cd68392553fe38a57bcb7c274a4c56ead", + "shasum": "" + }, + "require": { + "php": ">=8.0", + "utopia-php/cli": "0.11.*" + }, + "require-dev": { + "phpunit/phpunit": "^9.3", + "vimeo/psalm": "4.0.1" + }, + "type": "library", + "autoload": { + "psr-4": { + "Utopia\\Orchestration\\": "src/Orchestration" + } + }, + "autoload-dev": { + "psr-4": { + "Utopia\\Tests\\": "tests/Orchestration" + } + }, + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Eldad Fux", + "email": "eldad@appwrite.io" + } + ], + "description": "Lite & fast micro PHP abstraction library for container orchestration", + "keywords": [ + "docker", + "framework", + "kubernetes", + "orchestration", + "php", + "swarm", + "upf", + "utopia" + ], + "support": { + "source": "https://github.com/PineappleIOnic/orchestration/tree/dev" + }, + "time": "2021-07-23T08:43:52+00:00" + }, { "name": "utopia-php/preloader", "version": "0.2.4", @@ -2237,27 +2295,27 @@ "packages-dev": [ { "name": "amphp/amp", - "version": "v2.5.2", + "version": "v2.6.0", "source": { "type": "git", "url": "https://github.com/amphp/amp.git", - "reference": "efca2b32a7580087adb8aabbff6be1dc1bb924a9" + "reference": "caa95edeb1ca1bf7532e9118ede4a3c3126408cc" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/amphp/amp/zipball/efca2b32a7580087adb8aabbff6be1dc1bb924a9", - "reference": "efca2b32a7580087adb8aabbff6be1dc1bb924a9", + "url": "https://api.github.com/repos/amphp/amp/zipball/caa95edeb1ca1bf7532e9118ede4a3c3126408cc", + "reference": "caa95edeb1ca1bf7532e9118ede4a3c3126408cc", "shasum": "" }, "require": { - "php": ">=7" + "php": ">=7.1" }, "require-dev": { "amphp/php-cs-fixer-config": "dev-master", "amphp/phpunit-util": "^1", "ext-json": "*", "jetbrains/phpstorm-stubs": "^2019.3", - "phpunit/phpunit": "^6.0.9 | ^7", + "phpunit/phpunit": "^7 | ^8 | ^9", "psalm/phar": "^3.11@dev", "react/promise": "^2" }, @@ -2314,7 +2372,7 @@ "support": { "irc": "irc://irc.freenode.org/amphp", "issues": "https://github.com/amphp/amp/issues", - "source": "https://github.com/amphp/amp/tree/v2.5.2" + "source": "https://github.com/amphp/amp/tree/v2.6.0" }, "funding": [ { @@ -2322,7 +2380,7 @@ "type": "github" } ], - "time": "2021-01-10T17:06:37+00:00" + "time": "2021-07-16T20:06:06+00:00" }, { "name": "amphp/byte-stream", @@ -3117,16 +3175,16 @@ }, { "name": "nikic/php-parser", - "version": "v4.11.0", + "version": "v4.12.0", "source": { "type": "git", "url": "https://github.com/nikic/PHP-Parser.git", - "reference": "fe14cf3672a149364fb66dfe11bf6549af899f94" + "reference": "6608f01670c3cc5079e18c1dab1104e002579143" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/fe14cf3672a149364fb66dfe11bf6549af899f94", - "reference": "fe14cf3672a149364fb66dfe11bf6549af899f94", + "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/6608f01670c3cc5079e18c1dab1104e002579143", + "reference": "6608f01670c3cc5079e18c1dab1104e002579143", "shasum": "" }, "require": { @@ -3167,9 +3225,9 @@ ], "support": { "issues": "https://github.com/nikic/PHP-Parser/issues", - "source": "https://github.com/nikic/PHP-Parser/tree/v4.11.0" + "source": "https://github.com/nikic/PHP-Parser/tree/v4.12.0" }, - "time": "2021-07-03T13:36:55+00:00" + "time": "2021-07-21T10:44:31+00:00" }, { "name": "openlss/lib-array2xml", @@ -3226,16 +3284,16 @@ }, { "name": "phar-io/manifest", - "version": "2.0.1", + "version": "2.0.3", "source": { "type": "git", "url": "https://github.com/phar-io/manifest.git", - "reference": "85265efd3af7ba3ca4b2a2c34dbfc5788dd29133" + "reference": "97803eca37d319dfa7826cc2437fc020857acb53" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phar-io/manifest/zipball/85265efd3af7ba3ca4b2a2c34dbfc5788dd29133", - "reference": "85265efd3af7ba3ca4b2a2c34dbfc5788dd29133", + "url": "https://api.github.com/repos/phar-io/manifest/zipball/97803eca37d319dfa7826cc2437fc020857acb53", + "reference": "97803eca37d319dfa7826cc2437fc020857acb53", "shasum": "" }, "require": { @@ -3280,9 +3338,9 @@ "description": "Component for reading phar.io manifest information from a PHP Archive (PHAR)", "support": { "issues": "https://github.com/phar-io/manifest/issues", - "source": "https://github.com/phar-io/manifest/tree/master" + "source": "https://github.com/phar-io/manifest/tree/2.0.3" }, - "time": "2020-06-27T14:33:11+00:00" + "time": "2021-07-20T11:28:43+00:00" }, { "name": "phar-io/version", @@ -4882,6 +4940,7 @@ "type": "github" } ], + "abandoned": true, "time": "2020-09-28T06:45:17+00:00" }, { @@ -6066,7 +6125,9 @@ ], "aliases": [], "minimum-stability": "stable", - "stability-flags": [], + "stability-flags": { + "utopia-php/orchestration": 20 + }, "prefer-stable": false, "prefer-lowest": false, "platform": { @@ -6088,5 +6149,5 @@ "platform-overrides": { "php": "8.0" }, - "plugin-api-version": "2.0.0" + "plugin-api-version": "2.1.0" } From 9676f34d939fe15af3f2f7457e5d380c6b907377 Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Tue, 3 Aug 2021 11:17:22 +0100 Subject: [PATCH 06/20] Add suggestions --- app/workers/functions.php | 42 +++++++++--------- composer.lock | 90 +++++++++++++++++++-------------------- 2 files changed, 66 insertions(+), 66 deletions(-) diff --git a/app/workers/functions.php b/app/workers/functions.php index 4c5d2cd3a..310920de4 100644 --- a/app/workers/functions.php +++ b/app/workers/functions.php @@ -67,13 +67,13 @@ $response = $orchestration->list(['label' => 'appwrite-type=function']); $list = []; -array_map(function($value) use (&$list) { +foreach ($response as &$value) { $list[$value->getName()] = $value; -}, $response); +} $executionEnd = \microtime(true); -Console::info(count($list)." functions listed in " . ($executionEnd - $executionStart) . " seconds"); +Console::info(count($list).' functions listed in ' . ($executionEnd - $executionStart) . ' seconds'); /** * 1. Get event args - DONE @@ -383,29 +383,29 @@ class FunctionsV1 extends Worker $id = $orchestration->run( $runtime['image'], $container, - "", - array( - 'tail', + '', + ['tail', '-f', '/dev/null' - ), - "/usr/local/src", - array(), + ], + '/usr/local/src', + [], $vars, $tagPathTargetDir, - array( - "appwrite-type" => "function", - "appwrite-created" => "{$executionTime}" - )); + [ + 'appwrite-type' => 'function', + 'appwrite-created' => strval($executionTime) + ]); $tarStdout = ''; $tarStderr = ''; $untarSuccess = $orchestration->execute($container, - array( + [ 'sh', '-c', - 'mv /tmp/code.tar.gz /usr/local/src/code.tar.gz && tar -zxf /usr/local/src/code.tar.gz --strip 1 && rm /usr/local/src/code.tar.gz'), + 'mv /tmp/code.tar.gz /usr/local/src/code.tar.gz && tar -zxf /usr/local/src/code.tar.gz --strip 1 && rm /usr/local/src/code.tar.gz' + ], $tarStdout, $tarStderr, $vars, @@ -419,13 +419,13 @@ class FunctionsV1 extends Worker $list[$container] = new Container($container, $id, - "Up", - array( + 'Up', + [ 'appwrite-type' => 'function', - 'appwrite-created' => "{$executionTime}" - )); + 'appwrite-created' => strval($executionTime), + ]); - Console::info("Function created in " . ($executionEnd - $executionStart) . " seconds"); + Console::info('Function created in ' . ($executionEnd - $executionStart) . ' seconds'); } else { Console::info('Container is ready to run'); @@ -448,7 +448,7 @@ class FunctionsV1 extends Worker $executionTime = ($executionEnd - $executionStart); $functionStatus = $exitCode ? 'completed' : 'failed'; - Console::info("Function executed in " . ($executionEnd - $executionStart) . " seconds, status: {$functionStatus} "); + Console::info('Function executed in ' . ($executionEnd - $executionStart) . ' seconds, status: ' . $functionStatus); Authorization::disable(); diff --git a/composer.lock b/composer.lock index f6485ea18..32ec49699 100644 --- a/composer.lock +++ b/composer.lock @@ -1913,12 +1913,12 @@ "source": { "type": "git", "url": "https://github.com/PineappleIOnic/orchestration.git", - "reference": "b691209cd68392553fe38a57bcb7c274a4c56ead" + "reference": "ded89e7babf75ff1c3aefd4be1f4b235bd926835" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/PineappleIOnic/orchestration/zipball/b691209cd68392553fe38a57bcb7c274a4c56ead", - "reference": "b691209cd68392553fe38a57bcb7c274a4c56ead", + "url": "https://api.github.com/repos/PineappleIOnic/orchestration/zipball/ded89e7babf75ff1c3aefd4be1f4b235bd926835", + "reference": "ded89e7babf75ff1c3aefd4be1f4b235bd926835", "shasum": "" }, "require": { @@ -1963,7 +1963,7 @@ "support": { "source": "https://github.com/PineappleIOnic/orchestration/tree/dev" }, - "time": "2021-07-23T08:43:52+00:00" + "time": "2021-08-03T09:56:45+00:00" }, { "name": "utopia-php/preloader", @@ -2664,21 +2664,21 @@ }, { "name": "composer/xdebug-handler", - "version": "2.0.1", + "version": "2.0.2", "source": { "type": "git", "url": "https://github.com/composer/xdebug-handler.git", - "reference": "964adcdd3a28bf9ed5d9ac6450064e0d71ed7496" + "reference": "84674dd3a7575ba617f5a76d7e9e29a7d3891339" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/composer/xdebug-handler/zipball/964adcdd3a28bf9ed5d9ac6450064e0d71ed7496", - "reference": "964adcdd3a28bf9ed5d9ac6450064e0d71ed7496", + "url": "https://api.github.com/repos/composer/xdebug-handler/zipball/84674dd3a7575ba617f5a76d7e9e29a7d3891339", + "reference": "84674dd3a7575ba617f5a76d7e9e29a7d3891339", "shasum": "" }, "require": { "php": "^5.3.2 || ^7.0 || ^8.0", - "psr/log": "^1.0" + "psr/log": "^1 || ^2 || ^3" }, "require-dev": { "phpstan/phpstan": "^0.12.55", @@ -2708,7 +2708,7 @@ "support": { "irc": "irc://irc.freenode.org/composer", "issues": "https://github.com/composer/xdebug-handler/issues", - "source": "https://github.com/composer/xdebug-handler/tree/2.0.1" + "source": "https://github.com/composer/xdebug-handler/tree/2.0.2" }, "funding": [ { @@ -2724,7 +2724,7 @@ "type": "tidelift" } ], - "time": "2021-05-05T19:37:51+00:00" + "time": "2021-07-31T17:03:58+00:00" }, { "name": "dnoegel/php-xdg-base-dir", @@ -4940,7 +4940,6 @@ "type": "github" } ], - "abandoned": true, "time": "2020-09-28T06:45:17+00:00" }, { @@ -5106,16 +5105,16 @@ }, { "name": "symfony/console", - "version": "v5.3.2", + "version": "v5.3.6", "source": { "type": "git", "url": "https://github.com/symfony/console.git", - "reference": "649730483885ff2ca99ca0560ef0e5f6b03f2ac1" + "reference": "51b71afd6d2dc8f5063199357b9880cea8d8bfe2" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/console/zipball/649730483885ff2ca99ca0560ef0e5f6b03f2ac1", - "reference": "649730483885ff2ca99ca0560ef0e5f6b03f2ac1", + "url": "https://api.github.com/repos/symfony/console/zipball/51b71afd6d2dc8f5063199357b9880cea8d8bfe2", + "reference": "51b71afd6d2dc8f5063199357b9880cea8d8bfe2", "shasum": "" }, "require": { @@ -5123,11 +5122,12 @@ "symfony/deprecation-contracts": "^2.1", "symfony/polyfill-mbstring": "~1.0", "symfony/polyfill-php73": "^1.8", - "symfony/polyfill-php80": "^1.15", + "symfony/polyfill-php80": "^1.16", "symfony/service-contracts": "^1.1|^2", "symfony/string": "^5.1" }, "conflict": { + "psr/log": ">=3", "symfony/dependency-injection": "<4.4", "symfony/dotenv": "<5.1", "symfony/event-dispatcher": "<4.4", @@ -5135,10 +5135,10 @@ "symfony/process": "<4.4" }, "provide": { - "psr/log-implementation": "1.0" + "psr/log-implementation": "1.0|2.0" }, "require-dev": { - "psr/log": "~1.0", + "psr/log": "^1|^2", "symfony/config": "^4.4|^5.0", "symfony/dependency-injection": "^4.4|^5.0", "symfony/event-dispatcher": "^4.4|^5.0", @@ -5184,7 +5184,7 @@ "terminal" ], "support": { - "source": "https://github.com/symfony/console/tree/v5.3.2" + "source": "https://github.com/symfony/console/tree/v5.3.6" }, "funding": [ { @@ -5200,7 +5200,7 @@ "type": "tidelift" } ], - "time": "2021-06-12T09:42:48+00:00" + "time": "2021-07-27T19:10:22+00:00" }, { "name": "symfony/deprecation-contracts", @@ -5271,16 +5271,16 @@ }, { "name": "symfony/polyfill-intl-grapheme", - "version": "v1.23.0", + "version": "v1.23.1", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-intl-grapheme.git", - "reference": "24b72c6baa32c746a4d0840147c9715e42bb68ab" + "reference": "16880ba9c5ebe3642d1995ab866db29270b36535" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-intl-grapheme/zipball/24b72c6baa32c746a4d0840147c9715e42bb68ab", - "reference": "24b72c6baa32c746a4d0840147c9715e42bb68ab", + "url": "https://api.github.com/repos/symfony/polyfill-intl-grapheme/zipball/16880ba9c5ebe3642d1995ab866db29270b36535", + "reference": "16880ba9c5ebe3642d1995ab866db29270b36535", "shasum": "" }, "require": { @@ -5332,7 +5332,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-intl-grapheme/tree/v1.23.0" + "source": "https://github.com/symfony/polyfill-intl-grapheme/tree/v1.23.1" }, "funding": [ { @@ -5348,7 +5348,7 @@ "type": "tidelift" } ], - "time": "2021-05-27T09:17:38+00:00" + "time": "2021-05-27T12:26:48+00:00" }, { "name": "symfony/polyfill-intl-normalizer", @@ -5436,16 +5436,16 @@ }, { "name": "symfony/polyfill-mbstring", - "version": "v1.23.0", + "version": "v1.23.1", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-mbstring.git", - "reference": "2df51500adbaebdc4c38dea4c89a2e131c45c8a1" + "reference": "9174a3d80210dca8daa7f31fec659150bbeabfc6" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-mbstring/zipball/2df51500adbaebdc4c38dea4c89a2e131c45c8a1", - "reference": "2df51500adbaebdc4c38dea4c89a2e131c45c8a1", + "url": "https://api.github.com/repos/symfony/polyfill-mbstring/zipball/9174a3d80210dca8daa7f31fec659150bbeabfc6", + "reference": "9174a3d80210dca8daa7f31fec659150bbeabfc6", "shasum": "" }, "require": { @@ -5496,7 +5496,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-mbstring/tree/v1.23.0" + "source": "https://github.com/symfony/polyfill-mbstring/tree/v1.23.1" }, "funding": [ { @@ -5512,7 +5512,7 @@ "type": "tidelift" } ], - "time": "2021-05-27T09:27:20+00:00" + "time": "2021-05-27T12:26:48+00:00" }, { "name": "symfony/polyfill-php73", @@ -5595,16 +5595,16 @@ }, { "name": "symfony/polyfill-php80", - "version": "v1.23.0", + "version": "v1.23.1", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-php80.git", - "reference": "eca0bf41ed421bed1b57c4958bab16aa86b757d0" + "reference": "1100343ed1a92e3a38f9ae122fc0eb21602547be" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-php80/zipball/eca0bf41ed421bed1b57c4958bab16aa86b757d0", - "reference": "eca0bf41ed421bed1b57c4958bab16aa86b757d0", + "url": "https://api.github.com/repos/symfony/polyfill-php80/zipball/1100343ed1a92e3a38f9ae122fc0eb21602547be", + "reference": "1100343ed1a92e3a38f9ae122fc0eb21602547be", "shasum": "" }, "require": { @@ -5658,7 +5658,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-php80/tree/v1.23.0" + "source": "https://github.com/symfony/polyfill-php80/tree/v1.23.1" }, "funding": [ { @@ -5674,7 +5674,7 @@ "type": "tidelift" } ], - "time": "2021-02-19T12:13:01+00:00" + "time": "2021-07-28T13:41:28+00:00" }, { "name": "symfony/service-contracts", @@ -5840,16 +5840,16 @@ }, { "name": "theseer/tokenizer", - "version": "1.2.0", + "version": "1.2.1", "source": { "type": "git", "url": "https://github.com/theseer/tokenizer.git", - "reference": "75a63c33a8577608444246075ea0af0d052e452a" + "reference": "34a41e998c2183e22995f158c581e7b5e755ab9e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/theseer/tokenizer/zipball/75a63c33a8577608444246075ea0af0d052e452a", - "reference": "75a63c33a8577608444246075ea0af0d052e452a", + "url": "https://api.github.com/repos/theseer/tokenizer/zipball/34a41e998c2183e22995f158c581e7b5e755ab9e", + "reference": "34a41e998c2183e22995f158c581e7b5e755ab9e", "shasum": "" }, "require": { @@ -5878,7 +5878,7 @@ "description": "A small library for converting tokenized PHP source code into XML and potentially other formats", "support": { "issues": "https://github.com/theseer/tokenizer/issues", - "source": "https://github.com/theseer/tokenizer/tree/master" + "source": "https://github.com/theseer/tokenizer/tree/1.2.1" }, "funding": [ { @@ -5886,7 +5886,7 @@ "type": "github" } ], - "time": "2020-07-12T23:59:07+00:00" + "time": "2021-07-28T10:34:58+00:00" }, { "name": "twig/twig", From 46a9a4699a3daa64324f4ae6a046bb2db8ee8657 Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Tue, 3 Aug 2021 11:18:54 +0100 Subject: [PATCH 07/20] Update composer.lock --- composer.lock | 65 ++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 62 insertions(+), 3 deletions(-) diff --git a/composer.lock b/composer.lock index 9db0fcb30..f69c407df 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "63a89a825697892a52aa27d6819b5972", + "content-hash": "e69ba4e38368f60742ad646d923be1bb", "packages": [ { "name": "adhocore/jwt", @@ -1907,6 +1907,64 @@ }, "time": "2021-07-24T11:35:55+00:00" }, + { + "name": "utopia-php/orchestration", + "version": "dev-dev", + "source": { + "type": "git", + "url": "https://github.com/PineappleIOnic/orchestration.git", + "reference": "ded89e7babf75ff1c3aefd4be1f4b235bd926835" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/PineappleIOnic/orchestration/zipball/ded89e7babf75ff1c3aefd4be1f4b235bd926835", + "reference": "ded89e7babf75ff1c3aefd4be1f4b235bd926835", + "shasum": "" + }, + "require": { + "php": ">=8.0", + "utopia-php/cli": "0.11.*" + }, + "require-dev": { + "phpunit/phpunit": "^9.3", + "vimeo/psalm": "4.0.1" + }, + "type": "library", + "autoload": { + "psr-4": { + "Utopia\\Orchestration\\": "src/Orchestration" + } + }, + "autoload-dev": { + "psr-4": { + "Utopia\\Tests\\": "tests/Orchestration" + } + }, + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Eldad Fux", + "email": "eldad@appwrite.io" + } + ], + "description": "Lite & fast micro PHP abstraction library for container orchestration", + "keywords": [ + "docker", + "framework", + "kubernetes", + "orchestration", + "php", + "swarm", + "upf", + "utopia" + ], + "support": { + "source": "https://github.com/PineappleIOnic/orchestration/tree/dev" + }, + "time": "2021-08-03T09:56:45+00:00" + }, { "name": "utopia-php/preloader", "version": "0.2.4", @@ -4882,7 +4940,6 @@ "type": "github" } ], - "abandoned": true, "time": "2020-09-28T06:45:17+00:00" }, { @@ -6068,7 +6125,9 @@ ], "aliases": [], "minimum-stability": "stable", - "stability-flags": [], + "stability-flags": { + "utopia-php/orchestration": 20 + }, "prefer-stable": false, "prefer-lowest": false, "platform": { From 4c451c0835fb2942c6927bd21075251acdf58903 Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Wed, 4 Aug 2021 11:19:20 +0100 Subject: [PATCH 08/20] Add Suggestions --- app/workers/functions.php | 43 +++++++++++++++++++++++---------------- 1 file changed, 25 insertions(+), 18 deletions(-) diff --git a/app/workers/functions.php b/app/workers/functions.php index 310920de4..de0f44188 100644 --- a/app/workers/functions.php +++ b/app/workers/functions.php @@ -376,23 +376,23 @@ class FunctionsV1 extends Worker $orchestration->setMemory($memory); $orchestration->setSwap($swap); - \array_walk($vars, function (string &$value) { + foreach($vars as $key => $value) { $value = strval($value); - }); + } $id = $orchestration->run( - $runtime['image'], - $container, - '', - ['tail', + image: $runtime['image'], + name: $container, + entrypoint: '', + command: ['tail', '-f', '/dev/null' ], - '/usr/local/src', - [], - $vars, - $tagPathTargetDir, - [ + workdir: '/usr/local/src', + volumes: [], + vars: $vars, + mountFolder: $tagPathTargetDir, + labels: [ 'appwrite-type' => 'function', 'appwrite-created' => strval($executionTime) ]); @@ -400,16 +400,17 @@ class FunctionsV1 extends Worker $tarStdout = ''; $tarStderr = ''; - $untarSuccess = $orchestration->execute($container, - [ + $untarSuccess = $orchestration->execute( + name: $container, + command: [ 'sh', '-c', 'mv /tmp/code.tar.gz /usr/local/src/code.tar.gz && tar -zxf /usr/local/src/code.tar.gz --strip 1 && rm /usr/local/src/code.tar.gz' ], - $tarStdout, - $tarStderr, - $vars, - 60); + stdout: $tarStdout, + stderr: $tarStderr, + vars: $vars, + timeout: 60); if (!$untarSuccess) { throw new Exception('Error untarring code'); @@ -439,7 +440,13 @@ class FunctionsV1 extends Worker $exitCode = 0; try { - $exitCode = $orchestration->execute($container, $orchestration->parseCommandString($command), $stdout, $stderr, $vars, $function->getAttribute('timeout', (int) App::getEnv('_APP_FUNCTIONS_TIMEOUT', 900))); + $exitCode = $orchestration->execute( + name: $container, + command: $orchestration->parseCommandString($command), + stdout: $stdout, + stderr: $stderr, + vars: $vars, + timeout: $function->getAttribute('timeout', (int) App::getEnv('_APP_FUNCTIONS_TIMEOUT', 900))); } catch (TimeoutException $e) { $exitCode = 0; } From 1a8bed396aed738d67c8b9d8aea221dd9cc57c35 Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Wed, 11 Aug 2021 13:44:31 +0100 Subject: [PATCH 09/20] Fix env variable bug --- app/workers/functions.php | 8 ++++---- composer.lock | 20 +++++++++---------- .../Functions/FunctionsCustomClientTest.php | 1 - 3 files changed, 14 insertions(+), 15 deletions(-) diff --git a/app/workers/functions.php b/app/workers/functions.php index de0f44188..950fd7735 100644 --- a/app/workers/functions.php +++ b/app/workers/functions.php @@ -376,18 +376,18 @@ class FunctionsV1 extends Worker $orchestration->setMemory($memory); $orchestration->setSwap($swap); - foreach($vars as $key => $value) { + foreach($vars as &$value) { $value = strval($value); } $id = $orchestration->run( image: $runtime['image'], name: $container, - entrypoint: '', command: ['tail', - '-f', - '/dev/null' + '-f', + '/dev/null' ], + entrypoint: '', workdir: '/usr/local/src', volumes: [], vars: $vars, diff --git a/composer.lock b/composer.lock index f69c407df..802355eb2 100644 --- a/composer.lock +++ b/composer.lock @@ -1756,16 +1756,16 @@ }, { "name": "utopia-php/framework", - "version": "0.17.2", + "version": "0.17.3", "source": { "type": "git", "url": "https://github.com/utopia-php/framework.git", - "reference": "3cd5fa2a9e30040277861f4254c5ccd1b1600952" + "reference": "0274f6b3e49db2af0d702edf278ec7504dc99878" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/framework/zipball/3cd5fa2a9e30040277861f4254c5ccd1b1600952", - "reference": "3cd5fa2a9e30040277861f4254c5ccd1b1600952", + "url": "https://api.github.com/repos/utopia-php/framework/zipball/0274f6b3e49db2af0d702edf278ec7504dc99878", + "reference": "0274f6b3e49db2af0d702edf278ec7504dc99878", "shasum": "" }, "require": { @@ -1799,9 +1799,9 @@ ], "support": { "issues": "https://github.com/utopia-php/framework/issues", - "source": "https://github.com/utopia-php/framework/tree/0.17.2" + "source": "https://github.com/utopia-php/framework/tree/0.17.3" }, - "time": "2021-08-02T10:18:26+00:00" + "time": "2021-08-03T13:57:01+00:00" }, { "name": "utopia-php/image", @@ -1913,12 +1913,12 @@ "source": { "type": "git", "url": "https://github.com/PineappleIOnic/orchestration.git", - "reference": "ded89e7babf75ff1c3aefd4be1f4b235bd926835" + "reference": "d19c498731807aaadf583c58356011c8f39c53cf" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/PineappleIOnic/orchestration/zipball/ded89e7babf75ff1c3aefd4be1f4b235bd926835", - "reference": "ded89e7babf75ff1c3aefd4be1f4b235bd926835", + "url": "https://api.github.com/repos/PineappleIOnic/orchestration/zipball/d19c498731807aaadf583c58356011c8f39c53cf", + "reference": "d19c498731807aaadf583c58356011c8f39c53cf", "shasum": "" }, "require": { @@ -1963,7 +1963,7 @@ "support": { "source": "https://github.com/PineappleIOnic/orchestration/tree/dev" }, - "time": "2021-08-03T09:56:45+00:00" + "time": "2021-08-11T10:38:19+00:00" }, { "name": "utopia-php/preloader", diff --git a/tests/e2e/Services/Functions/FunctionsCustomClientTest.php b/tests/e2e/Services/Functions/FunctionsCustomClientTest.php index 454a149c3..4b52aa26c 100644 --- a/tests/e2e/Services/Functions/FunctionsCustomClientTest.php +++ b/tests/e2e/Services/Functions/FunctionsCustomClientTest.php @@ -196,7 +196,6 @@ class FunctionsCustomClientTest extends Scope ]); $output = json_decode($executions['body']['stdout'], true); - $this->assertEquals(200, $executions['headers']['status-code']); $this->assertEquals('completed', $executions['body']['status']); $this->assertEquals($functionId, $output['APPWRITE_FUNCTION_ID']); From 80fef728b42337a3682ba09bdb30f6cb9ffb8931 Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Wed, 11 Aug 2021 14:23:37 +0100 Subject: [PATCH 10/20] Implement UNIX Exit Code suggestion --- app/workers/functions.php | 8 ++++---- composer.lock | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/app/workers/functions.php b/app/workers/functions.php index 950fd7735..1a499c2d1 100644 --- a/app/workers/functions.php +++ b/app/workers/functions.php @@ -440,7 +440,7 @@ class FunctionsV1 extends Worker $exitCode = 0; try { - $exitCode = $orchestration->execute( + $exitCode = (int)!$orchestration->execute( name: $container, command: $orchestration->parseCommandString($command), stdout: $stdout, @@ -448,12 +448,12 @@ class FunctionsV1 extends Worker vars: $vars, timeout: $function->getAttribute('timeout', (int) App::getEnv('_APP_FUNCTIONS_TIMEOUT', 900))); } catch (TimeoutException $e) { - $exitCode = 0; + $exitCode = 124; } $executionEnd = \microtime(true); $executionTime = ($executionEnd - $executionStart); - $functionStatus = $exitCode ? 'completed' : 'failed'; + $functionStatus = ($exitCode === 0) ? 'completed' : 'failed'; Console::info('Function executed in ' . ($executionEnd - $executionStart) . ' seconds, status: ' . $functionStatus); @@ -462,7 +462,7 @@ class FunctionsV1 extends Worker $execution = $database->updateDocument(array_merge($execution->getArrayCopy(), [ 'tagId' => $tag->getId(), 'status' => $functionStatus, - 'exitCode' => ($exitCode ? 0 : 1), + 'exitCode' => ($exitCode === 0 ? 0 : 1), 'stdout' => \mb_substr($stdout, -4000), // log last 4000 chars output 'stderr' => \mb_substr($stderr, -4000), // log last 4000 chars output 'time' => $executionTime diff --git a/composer.lock b/composer.lock index 802355eb2..0105c0348 100644 --- a/composer.lock +++ b/composer.lock @@ -1913,12 +1913,12 @@ "source": { "type": "git", "url": "https://github.com/PineappleIOnic/orchestration.git", - "reference": "d19c498731807aaadf583c58356011c8f39c53cf" + "reference": "213d8796cd3b4a23b8d3728227b0e8ab1ceb4aea" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/PineappleIOnic/orchestration/zipball/d19c498731807aaadf583c58356011c8f39c53cf", - "reference": "d19c498731807aaadf583c58356011c8f39c53cf", + "url": "https://api.github.com/repos/PineappleIOnic/orchestration/zipball/213d8796cd3b4a23b8d3728227b0e8ab1ceb4aea", + "reference": "213d8796cd3b4a23b8d3728227b0e8ab1ceb4aea", "shasum": "" }, "require": { @@ -1963,7 +1963,7 @@ "support": { "source": "https://github.com/PineappleIOnic/orchestration/tree/dev" }, - "time": "2021-08-11T10:38:19+00:00" + "time": "2021-08-11T12:45:16+00:00" }, { "name": "utopia-php/preloader", From eb3db0ddaefae3c2f16a5e28db6c5ea9fa9ada52 Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Wed, 11 Aug 2021 14:24:33 +0100 Subject: [PATCH 11/20] Update functions.php --- app/workers/functions.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/workers/functions.php b/app/workers/functions.php index 1a499c2d1..d1cff1f12 100644 --- a/app/workers/functions.php +++ b/app/workers/functions.php @@ -413,7 +413,7 @@ class FunctionsV1 extends Worker timeout: 60); if (!$untarSuccess) { - throw new Exception('Error untarring code'); + throw new Exception('Failed to extract tar: '.$stderr); } $executionEnd = \microtime(true); From bd537e69925f6f908a170fd1071db9a53adfab03 Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Fri, 13 Aug 2021 09:32:27 +0100 Subject: [PATCH 12/20] Update Tests --- app/workers/functions.php | 32 ++++++++++++++------------------ composer.lock | 8 ++++---- 2 files changed, 18 insertions(+), 22 deletions(-) diff --git a/app/workers/functions.php b/app/workers/functions.php index d1cff1f12..a5865291d 100644 --- a/app/workers/functions.php +++ b/app/workers/functions.php @@ -16,7 +16,7 @@ use Utopia\Config\Config; use Utopia\Orchestration\Orchestration; use Utopia\Orchestration\Adapter\DockerAPI; use Utopia\Orchestration\Container; -use Utopia\Orchestration\Exceptions\TimeoutException; +use Utopia\Orchestration\Exception\Timeout as TimeoutException; require_once __DIR__.'/../workers.php'; @@ -29,7 +29,8 @@ $runtimes = Config::getParam('runtimes'); $dockerUser = App::getEnv('DOCKERHUB_PULL_USERNAME', null); $dockerPass = App::getEnv('DOCKERHUB_PULL_PASSWORD', null); -$orchestration = new Orchestration(new DockerAPI($dockerUser, $dockerPass)); +$dockerEmail = App::getEnv('DOCKERHUB_PULL_EMAIL', null); +$orchestration = new Orchestration(new DockerAPI($dockerUser, $dockerPass, $dockerEmail)); /** * Warmup Docker Images @@ -67,7 +68,7 @@ $response = $orchestration->list(['label' => 'appwrite-type=function']); $list = []; -foreach ($response as &$value) { +foreach ($response as $value) { $list[$value->getName()] = $value; } @@ -368,13 +369,10 @@ class FunctionsV1 extends Worker $executionStart = \microtime(true); $executionTime = \time(); - $cpus = App::getEnv('_APP_FUNCTIONS_CPUS', ''); - $memory = App::getEnv('_APP_FUNCTIONS_MEMORY', ''); - $swap = App::getEnv('_APP_FUNCTIONS_MEMORY_SWAP', ''); - $orchestration->setCpus($cpus); - $orchestration->setMemory($memory); - $orchestration->setSwap($swap); + $orchestration->setCpus(App::getEnv('_APP_FUNCTIONS_CPUS', '')); + $orchestration->setMemory(App::getEnv('_APP_FUNCTIONS_MEMORY', '')); + $orchestration->setSwap(App::getEnv('_APP_FUNCTIONS_MEMORY_SWAP', '')); foreach($vars as &$value) { $value = strval($value); @@ -397,8 +395,8 @@ class FunctionsV1 extends Worker 'appwrite-created' => strval($executionTime) ]); - $tarStdout = ''; - $tarStderr = ''; + $untarStdout = ''; + $untarStderr = ''; $untarSuccess = $orchestration->execute( name: $container, @@ -407,20 +405,18 @@ class FunctionsV1 extends Worker '-c', 'mv /tmp/code.tar.gz /usr/local/src/code.tar.gz && tar -zxf /usr/local/src/code.tar.gz --strip 1 && rm /usr/local/src/code.tar.gz' ], - stdout: $tarStdout, - stderr: $tarStderr, + stdout: $untarStdout, + stderr: $untarStderr, vars: $vars, timeout: 60); if (!$untarSuccess) { - throw new Exception('Failed to extract tar: '.$stderr); + throw new Exception('Failed to extract tar: '.$untarStderr); } $executionEnd = \microtime(true); - $list[$container] = new Container($container, - $id, - 'Up', + $list[$container] = new Container($container, $id, 'Up', [ 'appwrite-type' => 'function', 'appwrite-created' => strval($executionTime), @@ -462,7 +458,7 @@ class FunctionsV1 extends Worker $execution = $database->updateDocument(array_merge($execution->getArrayCopy(), [ 'tagId' => $tag->getId(), 'status' => $functionStatus, - 'exitCode' => ($exitCode === 0 ? 0 : 1), + 'exitCode' => $exitCode, 'stdout' => \mb_substr($stdout, -4000), // log last 4000 chars output 'stderr' => \mb_substr($stderr, -4000), // log last 4000 chars output 'time' => $executionTime diff --git a/composer.lock b/composer.lock index 0105c0348..ab9ed154e 100644 --- a/composer.lock +++ b/composer.lock @@ -1913,12 +1913,12 @@ "source": { "type": "git", "url": "https://github.com/PineappleIOnic/orchestration.git", - "reference": "213d8796cd3b4a23b8d3728227b0e8ab1ceb4aea" + "reference": "f8f31b043b8f4bbd423cb365f84e89364e9a349a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/PineappleIOnic/orchestration/zipball/213d8796cd3b4a23b8d3728227b0e8ab1ceb4aea", - "reference": "213d8796cd3b4a23b8d3728227b0e8ab1ceb4aea", + "url": "https://api.github.com/repos/PineappleIOnic/orchestration/zipball/f8f31b043b8f4bbd423cb365f84e89364e9a349a", + "reference": "f8f31b043b8f4bbd423cb365f84e89364e9a349a", "shasum": "" }, "require": { @@ -1963,7 +1963,7 @@ "support": { "source": "https://github.com/PineappleIOnic/orchestration/tree/dev" }, - "time": "2021-08-11T12:45:16+00:00" + "time": "2021-08-13T08:21:13+00:00" }, { "name": "utopia-php/preloader", From 5080160f09aa9574bd4a00379a71fe13c0f6b717 Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Fri, 13 Aug 2021 09:34:21 +0100 Subject: [PATCH 13/20] Revert "Update Tests" This reverts commit bd537e69925f6f908a170fd1071db9a53adfab03. --- app/workers/functions.php | 32 ++++++++++++++++++-------------- composer.lock | 8 ++++---- 2 files changed, 22 insertions(+), 18 deletions(-) diff --git a/app/workers/functions.php b/app/workers/functions.php index a5865291d..d1cff1f12 100644 --- a/app/workers/functions.php +++ b/app/workers/functions.php @@ -16,7 +16,7 @@ use Utopia\Config\Config; use Utopia\Orchestration\Orchestration; use Utopia\Orchestration\Adapter\DockerAPI; use Utopia\Orchestration\Container; -use Utopia\Orchestration\Exception\Timeout as TimeoutException; +use Utopia\Orchestration\Exceptions\TimeoutException; require_once __DIR__.'/../workers.php'; @@ -29,8 +29,7 @@ $runtimes = Config::getParam('runtimes'); $dockerUser = App::getEnv('DOCKERHUB_PULL_USERNAME', null); $dockerPass = App::getEnv('DOCKERHUB_PULL_PASSWORD', null); -$dockerEmail = App::getEnv('DOCKERHUB_PULL_EMAIL', null); -$orchestration = new Orchestration(new DockerAPI($dockerUser, $dockerPass, $dockerEmail)); +$orchestration = new Orchestration(new DockerAPI($dockerUser, $dockerPass)); /** * Warmup Docker Images @@ -68,7 +67,7 @@ $response = $orchestration->list(['label' => 'appwrite-type=function']); $list = []; -foreach ($response as $value) { +foreach ($response as &$value) { $list[$value->getName()] = $value; } @@ -369,10 +368,13 @@ class FunctionsV1 extends Worker $executionStart = \microtime(true); $executionTime = \time(); + $cpus = App::getEnv('_APP_FUNCTIONS_CPUS', ''); + $memory = App::getEnv('_APP_FUNCTIONS_MEMORY', ''); + $swap = App::getEnv('_APP_FUNCTIONS_MEMORY_SWAP', ''); - $orchestration->setCpus(App::getEnv('_APP_FUNCTIONS_CPUS', '')); - $orchestration->setMemory(App::getEnv('_APP_FUNCTIONS_MEMORY', '')); - $orchestration->setSwap(App::getEnv('_APP_FUNCTIONS_MEMORY_SWAP', '')); + $orchestration->setCpus($cpus); + $orchestration->setMemory($memory); + $orchestration->setSwap($swap); foreach($vars as &$value) { $value = strval($value); @@ -395,8 +397,8 @@ class FunctionsV1 extends Worker 'appwrite-created' => strval($executionTime) ]); - $untarStdout = ''; - $untarStderr = ''; + $tarStdout = ''; + $tarStderr = ''; $untarSuccess = $orchestration->execute( name: $container, @@ -405,18 +407,20 @@ class FunctionsV1 extends Worker '-c', 'mv /tmp/code.tar.gz /usr/local/src/code.tar.gz && tar -zxf /usr/local/src/code.tar.gz --strip 1 && rm /usr/local/src/code.tar.gz' ], - stdout: $untarStdout, - stderr: $untarStderr, + stdout: $tarStdout, + stderr: $tarStderr, vars: $vars, timeout: 60); if (!$untarSuccess) { - throw new Exception('Failed to extract tar: '.$untarStderr); + throw new Exception('Failed to extract tar: '.$stderr); } $executionEnd = \microtime(true); - $list[$container] = new Container($container, $id, 'Up', + $list[$container] = new Container($container, + $id, + 'Up', [ 'appwrite-type' => 'function', 'appwrite-created' => strval($executionTime), @@ -458,7 +462,7 @@ class FunctionsV1 extends Worker $execution = $database->updateDocument(array_merge($execution->getArrayCopy(), [ 'tagId' => $tag->getId(), 'status' => $functionStatus, - 'exitCode' => $exitCode, + 'exitCode' => ($exitCode === 0 ? 0 : 1), 'stdout' => \mb_substr($stdout, -4000), // log last 4000 chars output 'stderr' => \mb_substr($stderr, -4000), // log last 4000 chars output 'time' => $executionTime diff --git a/composer.lock b/composer.lock index ab9ed154e..0105c0348 100644 --- a/composer.lock +++ b/composer.lock @@ -1913,12 +1913,12 @@ "source": { "type": "git", "url": "https://github.com/PineappleIOnic/orchestration.git", - "reference": "f8f31b043b8f4bbd423cb365f84e89364e9a349a" + "reference": "213d8796cd3b4a23b8d3728227b0e8ab1ceb4aea" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/PineappleIOnic/orchestration/zipball/f8f31b043b8f4bbd423cb365f84e89364e9a349a", - "reference": "f8f31b043b8f4bbd423cb365f84e89364e9a349a", + "url": "https://api.github.com/repos/PineappleIOnic/orchestration/zipball/213d8796cd3b4a23b8d3728227b0e8ab1ceb4aea", + "reference": "213d8796cd3b4a23b8d3728227b0e8ab1ceb4aea", "shasum": "" }, "require": { @@ -1963,7 +1963,7 @@ "support": { "source": "https://github.com/PineappleIOnic/orchestration/tree/dev" }, - "time": "2021-08-13T08:21:13+00:00" + "time": "2021-08-11T12:45:16+00:00" }, { "name": "utopia-php/preloader", From 44ac60a70a3f7a0811bdad76106e35b85779aae0 Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Fri, 13 Aug 2021 10:00:20 +0100 Subject: [PATCH 14/20] Implement Suggestions --- app/workers/functions.php | 42 +++++++++---------- composer.lock | 8 ++-- .../Functions/FunctionsCustomServerTest.php | 2 +- 3 files changed, 25 insertions(+), 27 deletions(-) diff --git a/app/workers/functions.php b/app/workers/functions.php index d1cff1f12..731f8b0bb 100644 --- a/app/workers/functions.php +++ b/app/workers/functions.php @@ -16,7 +16,7 @@ use Utopia\Config\Config; use Utopia\Orchestration\Orchestration; use Utopia\Orchestration\Adapter\DockerAPI; use Utopia\Orchestration\Container; -use Utopia\Orchestration\Exceptions\TimeoutException; +use Utopia\Orchestration\Exception\Timeout as TimeoutException; require_once __DIR__.'/../workers.php'; @@ -29,7 +29,8 @@ $runtimes = Config::getParam('runtimes'); $dockerUser = App::getEnv('DOCKERHUB_PULL_USERNAME', null); $dockerPass = App::getEnv('DOCKERHUB_PULL_PASSWORD', null); -$orchestration = new Orchestration(new DockerAPI($dockerUser, $dockerPass)); +$dockerEmail = App::getEnv('DOCKERHUB_PULL_EMAIL', null); +$orchestration = new Orchestration(new DockerAPI($dockerUser, $dockerPass, $dockerEmail)); /** * Warmup Docker Images @@ -40,11 +41,13 @@ Co\run(function() use ($runtimes, $orchestration) { // Warmup: make sure images foreach($runtimes as $runtime) { go(function() use ($runtime, $orchestration) { Console::info('Warming up '.$runtime['name'].' '.$runtime['version'].' environment...'); - try { - $orchestration->pull($runtime['image']); + + $response = $orchestration->pull($runtime['image']); + + if ($response) { Console::success("Successfully Warmed up {$runtime['name']} {$runtime['version']}!"); - } catch (Exception $e) { - Console::error($e); + } else { + Console::error("Failed to Warmup {$runtime['name']} {$runtime['version']}!"); } }); } @@ -67,7 +70,7 @@ $response = $orchestration->list(['label' => 'appwrite-type=function']); $list = []; -foreach ($response as &$value) { +foreach ($response as $value) { $list[$value->getName()] = $value; } @@ -368,13 +371,10 @@ class FunctionsV1 extends Worker $executionStart = \microtime(true); $executionTime = \time(); - $cpus = App::getEnv('_APP_FUNCTIONS_CPUS', ''); - $memory = App::getEnv('_APP_FUNCTIONS_MEMORY', ''); - $swap = App::getEnv('_APP_FUNCTIONS_MEMORY_SWAP', ''); - $orchestration->setCpus($cpus); - $orchestration->setMemory($memory); - $orchestration->setSwap($swap); + $orchestration->setCpus(App::getEnv('_APP_FUNCTIONS_CPUS', '')); + $orchestration->setMemory(App::getEnv('_APP_FUNCTIONS_MEMORY', '')); + $orchestration->setSwap(App::getEnv('_APP_FUNCTIONS_MEMORY_SWAP', '')); foreach($vars as &$value) { $value = strval($value); @@ -397,8 +397,8 @@ class FunctionsV1 extends Worker 'appwrite-created' => strval($executionTime) ]); - $tarStdout = ''; - $tarStderr = ''; + $untarStdout = ''; + $untarStderr = ''; $untarSuccess = $orchestration->execute( name: $container, @@ -407,20 +407,18 @@ class FunctionsV1 extends Worker '-c', 'mv /tmp/code.tar.gz /usr/local/src/code.tar.gz && tar -zxf /usr/local/src/code.tar.gz --strip 1 && rm /usr/local/src/code.tar.gz' ], - stdout: $tarStdout, - stderr: $tarStderr, + stdout: $untarStdout, + stderr: $untarStderr, vars: $vars, timeout: 60); if (!$untarSuccess) { - throw new Exception('Failed to extract tar: '.$stderr); + throw new Exception('Failed to extract tar: '.$untarStderr); } $executionEnd = \microtime(true); - $list[$container] = new Container($container, - $id, - 'Up', + $list[$container] = new Container($container, $id, 'Up', [ 'appwrite-type' => 'function', 'appwrite-created' => strval($executionTime), @@ -462,7 +460,7 @@ class FunctionsV1 extends Worker $execution = $database->updateDocument(array_merge($execution->getArrayCopy(), [ 'tagId' => $tag->getId(), 'status' => $functionStatus, - 'exitCode' => ($exitCode === 0 ? 0 : 1), + 'exitCode' => $exitCode, 'stdout' => \mb_substr($stdout, -4000), // log last 4000 chars output 'stderr' => \mb_substr($stderr, -4000), // log last 4000 chars output 'time' => $executionTime diff --git a/composer.lock b/composer.lock index 0105c0348..6fec3d7e2 100644 --- a/composer.lock +++ b/composer.lock @@ -1913,12 +1913,12 @@ "source": { "type": "git", "url": "https://github.com/PineappleIOnic/orchestration.git", - "reference": "213d8796cd3b4a23b8d3728227b0e8ab1ceb4aea" + "reference": "e888d0ae03fdaad38b37631cc8a7d75ac0d6c692" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/PineappleIOnic/orchestration/zipball/213d8796cd3b4a23b8d3728227b0e8ab1ceb4aea", - "reference": "213d8796cd3b4a23b8d3728227b0e8ab1ceb4aea", + "url": "https://api.github.com/repos/PineappleIOnic/orchestration/zipball/e888d0ae03fdaad38b37631cc8a7d75ac0d6c692", + "reference": "e888d0ae03fdaad38b37631cc8a7d75ac0d6c692", "shasum": "" }, "require": { @@ -1963,7 +1963,7 @@ "support": { "source": "https://github.com/PineappleIOnic/orchestration/tree/dev" }, - "time": "2021-08-11T12:45:16+00:00" + "time": "2021-08-13T08:34:33+00:00" }, { "name": "utopia-php/preloader", diff --git a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php index 775eca06e..58740150d 100644 --- a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php +++ b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php @@ -518,7 +518,7 @@ class FunctionsCustomServerTest extends Scope $this->assertEquals($executions['body']['executions'][0]['$id'], $executionId); $this->assertEquals($executions['body']['executions'][0]['trigger'], 'http'); $this->assertEquals($executions['body']['executions'][0]['status'], 'failed'); - $this->assertEquals($executions['body']['executions'][0]['exitCode'], 1); + $this->assertEquals($executions['body']['executions'][0]['exitCode'], 124); $this->assertGreaterThan(2, $executions['body']['executions'][0]['time']); $this->assertLessThan(3, $executions['body']['executions'][0]['time']); $this->assertEquals($executions['body']['executions'][0]['stdout'], ''); From 054eadc6f66a531fc59739e61cc9f9f12f8b6a3c Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Fri, 13 Aug 2021 15:16:05 +0100 Subject: [PATCH 15/20] Implement Suggestions + Added documentation for environment variables + Added changelog + Changed how we handle a error while removing a container that may or may not exist during execution --- CHANGES.md | 5 +++++ app/config/variables.php | 27 +++++++++++++++++++++++++++ app/workers/functions.php | 8 ++++++-- 3 files changed, 38 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index c6e74d852..cc64f1e18 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,8 @@ +# Version 0.10.0 (Not Released Yet) +- Switch from using Docker CLI to Docker API by intergrating [utopia-php/orchestration](https://github.com/utopia-php/orchestration) +- Added DOCKERHUB_PULL_USERNAME, DOCKERHUB_PULL_PASSWORD and DOCKERHUB_PULL_EMAIL env variables for pulling from + private DockerHub repos + # Version 0.9.3 ## Bugs diff --git a/app/config/variables.php b/app/config/variables.php index df7346cd5..bb505fd6b 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -462,6 +462,33 @@ return [ 'question' => '', 'filter' => '' ], + [ + 'name' => 'DOCKERHUB_PULL_USERNAME', + 'description' => 'The username for hub.docker.com. This variable is used to pull images from hub.docker.com.', + 'introduction' => '0.10.0', + 'default' => '', + 'required' => false, + 'question' => '', + 'filter' => '' + ], + [ + 'name' => 'DOCKERHUB_PULL_PASSWORD', + 'description' => 'The password for hub.docker.com. This variable is used to pull images from hub.docker.com.', + 'introduction' => '0.10.0', + 'default' => '', + 'required' => false, + 'question' => '', + 'filter' => '' + ], + [ + 'name' => 'DOCKERHUB_PULL_EMAIL', + 'description' => 'The email for hub.docker.com. This variable is used to pull images from hub.docker.com.', + 'introduction' => '0.10.0', + 'default' => '', + 'required' => false, + 'question' => '', + 'filter' => '' + ], ], [ 'category' => 'Maintenance', diff --git a/app/workers/functions.php b/app/workers/functions.php index 731f8b0bb..3393c8365 100644 --- a/app/workers/functions.php +++ b/app/workers/functions.php @@ -347,11 +347,15 @@ class FunctionsV1 extends Worker } } - if(isset($list[$container]) && !(\substr($list[$container]->getStatus(), 0, 2) === 'Up')) { // Remove conatiner if not online + if(isset($list[$container]) && !(\substr($list[$container]->getStatus(), 0, 2) === 'Up')) { // Remove conatiner if not online $stdout = ''; $stderr = ''; - $orchestration->remove($container); + try { + $orchestration->remove($container); + } catch (Exception $e) { + Console::warning('Failed to remove container: '.$e->getMessage()); + } unset($list[$container]); } From 0d69db30b0e8495d2d6c34e85b7dcef7a3244701 Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Fri, 13 Aug 2021 15:49:12 +0100 Subject: [PATCH 16/20] Start Travis From fa15b37433507bac8d8aa1574c508b26543e8359 Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Mon, 16 Aug 2021 09:19:09 +0100 Subject: [PATCH 17/20] Add defaults for environment variables --- app/workers/functions.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/workers/functions.php b/app/workers/functions.php index 3393c8365..4540ac165 100644 --- a/app/workers/functions.php +++ b/app/workers/functions.php @@ -376,9 +376,9 @@ class FunctionsV1 extends Worker $executionStart = \microtime(true); $executionTime = \time(); - $orchestration->setCpus(App::getEnv('_APP_FUNCTIONS_CPUS', '')); - $orchestration->setMemory(App::getEnv('_APP_FUNCTIONS_MEMORY', '')); - $orchestration->setSwap(App::getEnv('_APP_FUNCTIONS_MEMORY_SWAP', '')); + $orchestration->setCpus(App::getEnv('_APP_FUNCTIONS_CPUS', '1')); + $orchestration->setMemory(App::getEnv('_APP_FUNCTIONS_MEMORY', '256')); + $orchestration->setSwap(App::getEnv('_APP_FUNCTIONS_MEMORY_SWAP', '256')); foreach($vars as &$value) { $value = strval($value); From c662d9bb2a077e047f4d84c246054be47a64b882 Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Mon, 16 Aug 2021 14:04:32 +0100 Subject: [PATCH 18/20] Update to use packagist repo --- composer.json | 8 +------- composer.lock | 27 +++++++++++---------------- 2 files changed, 12 insertions(+), 23 deletions(-) diff --git a/composer.json b/composer.json index aca0c31f3..2b022cc3a 100644 --- a/composer.json +++ b/composer.json @@ -9,12 +9,6 @@ "email": "eldad@appwrite.io" } ], - "repositories": [ - { - "type": "vcs", - "url": "https://github.com/PineappleIOnic/orchestration" - } - ], "autoload": { "psr-4": { "Appwrite\\": "src/Appwrite" @@ -60,7 +54,7 @@ "utopia-php/image": "0.5.*", "resque/php-resque": "1.3.6", "matomo/device-detector": "4.2.3", - "utopia-php/orchestration": "dev-dev", + "utopia-php/orchestration": "0.2.0", "dragonmantank/cron-expression": "3.1.0", "influxdb/influxdb-php": "1.15.2", "phpmailer/phpmailer": "6.5.0", diff --git a/composer.lock b/composer.lock index 6fec3d7e2..95f6b3e16 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "e69ba4e38368f60742ad646d923be1bb", + "content-hash": "0f073f85ca3b161b6f1d293bd8ebe358", "packages": [ { "name": "adhocore/jwt", @@ -1909,16 +1909,16 @@ }, { "name": "utopia-php/orchestration", - "version": "dev-dev", + "version": "0.2.0", "source": { "type": "git", - "url": "https://github.com/PineappleIOnic/orchestration.git", - "reference": "e888d0ae03fdaad38b37631cc8a7d75ac0d6c692" + "url": "https://github.com/utopia-php/orchestration.git", + "reference": "de10509017768cf2b62363bb39912002ab41dafb" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/PineappleIOnic/orchestration/zipball/e888d0ae03fdaad38b37631cc8a7d75ac0d6c692", - "reference": "e888d0ae03fdaad38b37631cc8a7d75ac0d6c692", + "url": "https://api.github.com/repos/utopia-php/orchestration/zipball/de10509017768cf2b62363bb39912002ab41dafb", + "reference": "de10509017768cf2b62363bb39912002ab41dafb", "shasum": "" }, "require": { @@ -1935,11 +1935,7 @@ "Utopia\\Orchestration\\": "src/Orchestration" } }, - "autoload-dev": { - "psr-4": { - "Utopia\\Tests\\": "tests/Orchestration" - } - }, + "notification-url": "https://packagist.org/downloads/", "license": [ "MIT" ], @@ -1961,9 +1957,10 @@ "utopia" ], "support": { - "source": "https://github.com/PineappleIOnic/orchestration/tree/dev" + "issues": "https://github.com/utopia-php/orchestration/issues", + "source": "https://github.com/utopia-php/orchestration/tree/0.2.0" }, - "time": "2021-08-13T08:34:33+00:00" + "time": "2021-08-16T12:52:42+00:00" }, { "name": "utopia-php/preloader", @@ -6125,9 +6122,7 @@ ], "aliases": [], "minimum-stability": "stable", - "stability-flags": { - "utopia-php/orchestration": 20 - }, + "stability-flags": [], "prefer-stable": false, "prefer-lowest": false, "platform": { From 38e16246368eb994ef42ff5a1d76c61675494da3 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Tue, 31 Aug 2021 20:52:47 +0200 Subject: [PATCH 19/20] fix(functions): flag execution errors as failed --- app/workers/functions.php | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/app/workers/functions.php b/app/workers/functions.php index 4540ac165..55387422e 100644 --- a/app/workers/functions.php +++ b/app/workers/functions.php @@ -16,6 +16,7 @@ use Utopia\Config\Config; use Utopia\Orchestration\Orchestration; use Utopia\Orchestration\Adapter\DockerAPI; use Utopia\Orchestration\Container; +use Utopia\Orchestration\Exception\Orchestration as OrchestrationException; use Utopia\Orchestration\Exception\Timeout as TimeoutException; require_once __DIR__.'/../workers.php'; @@ -433,7 +434,7 @@ class FunctionsV1 extends Worker else { Console::info('Container is ready to run'); } - + $stdout = ''; $stderr = ''; @@ -451,6 +452,9 @@ class FunctionsV1 extends Worker timeout: $function->getAttribute('timeout', (int) App::getEnv('_APP_FUNCTIONS_TIMEOUT', 900))); } catch (TimeoutException $e) { $exitCode = 124; + } catch (OrchestrationException $e) { + $stderr = $e->getMessage(); + $exitCode = 1; } $executionEnd = \microtime(true); @@ -460,7 +464,7 @@ class FunctionsV1 extends Worker Console::info('Function executed in ' . ($executionEnd - $executionStart) . ' seconds, status: ' . $functionStatus); Authorization::disable(); - + $execution = $database->updateDocument(array_merge($execution->getArrayCopy(), [ 'tagId' => $tag->getId(), 'status' => $functionStatus, From 9755663547e4821570bc4808b66791c51efd244c Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Tue, 31 Aug 2021 21:08:29 +0200 Subject: [PATCH 20/20] chore(composer): update lock file --- composer.lock | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/composer.lock b/composer.lock index 569ebaf16..d1fa6eecf 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "45963af754680568d89330a4f37c40d1", + "content-hash": "0a782184d016e458ff18f20a2c49ccfb", "packages": [ { "name": "adhocore/jwt", @@ -2507,16 +2507,16 @@ }, { "name": "composer/package-versions-deprecated", - "version": "1.11.99.2", + "version": "1.11.99.3", "source": { "type": "git", "url": "https://github.com/composer/package-versions-deprecated.git", - "reference": "c6522afe5540d5fc46675043d3ed5a45a740b27c" + "reference": "fff576ac850c045158a250e7e27666e146e78d18" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/composer/package-versions-deprecated/zipball/c6522afe5540d5fc46675043d3ed5a45a740b27c", - "reference": "c6522afe5540d5fc46675043d3ed5a45a740b27c", + "url": "https://api.github.com/repos/composer/package-versions-deprecated/zipball/fff576ac850c045158a250e7e27666e146e78d18", + "reference": "fff576ac850c045158a250e7e27666e146e78d18", "shasum": "" }, "require": { @@ -2560,7 +2560,7 @@ "description": "Composer plugin that provides efficient querying for installed package versions (no runtime IO)", "support": { "issues": "https://github.com/composer/package-versions-deprecated/issues", - "source": "https://github.com/composer/package-versions-deprecated/tree/1.11.99.2" + "source": "https://github.com/composer/package-versions-deprecated/tree/1.11.99.3" }, "funding": [ { @@ -2576,7 +2576,7 @@ "type": "tidelift" } ], - "time": "2021-05-24T07:46:03+00:00" + "time": "2021-08-17T13:49:14+00:00" }, { "name": "composer/semver", @@ -5102,16 +5102,16 @@ }, { "name": "symfony/console", - "version": "v5.3.6", + "version": "v5.3.7", "source": { "type": "git", "url": "https://github.com/symfony/console.git", - "reference": "51b71afd6d2dc8f5063199357b9880cea8d8bfe2" + "reference": "8b1008344647462ae6ec57559da166c2bfa5e16a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/console/zipball/51b71afd6d2dc8f5063199357b9880cea8d8bfe2", - "reference": "51b71afd6d2dc8f5063199357b9880cea8d8bfe2", + "url": "https://api.github.com/repos/symfony/console/zipball/8b1008344647462ae6ec57559da166c2bfa5e16a", + "reference": "8b1008344647462ae6ec57559da166c2bfa5e16a", "shasum": "" }, "require": { @@ -5181,7 +5181,7 @@ "terminal" ], "support": { - "source": "https://github.com/symfony/console/tree/v5.3.6" + "source": "https://github.com/symfony/console/tree/v5.3.7" }, "funding": [ { @@ -5197,7 +5197,7 @@ "type": "tidelift" } ], - "time": "2021-07-27T19:10:22+00:00" + "time": "2021-08-25T20:02:16+00:00" }, { "name": "symfony/deprecation-contracts", @@ -5754,16 +5754,16 @@ }, { "name": "symfony/string", - "version": "v5.3.3", + "version": "v5.3.7", "source": { "type": "git", "url": "https://github.com/symfony/string.git", - "reference": "bd53358e3eccec6a670b5f33ab680d8dbe1d4ae1" + "reference": "8d224396e28d30f81969f083a58763b8b9ceb0a5" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/string/zipball/bd53358e3eccec6a670b5f33ab680d8dbe1d4ae1", - "reference": "bd53358e3eccec6a670b5f33ab680d8dbe1d4ae1", + "url": "https://api.github.com/repos/symfony/string/zipball/8d224396e28d30f81969f083a58763b8b9ceb0a5", + "reference": "8d224396e28d30f81969f083a58763b8b9ceb0a5", "shasum": "" }, "require": { @@ -5817,7 +5817,7 @@ "utf8" ], "support": { - "source": "https://github.com/symfony/string/tree/v5.3.3" + "source": "https://github.com/symfony/string/tree/v5.3.7" }, "funding": [ { @@ -5833,7 +5833,7 @@ "type": "tidelift" } ], - "time": "2021-06-27T11:44:38+00:00" + "time": "2021-08-26T08:00:08+00:00" }, { "name": "theseer/tokenizer", @@ -6144,5 +6144,5 @@ "platform-overrides": { "php": "8.0" }, - "plugin-api-version": "2.1.0" + "plugin-api-version": "2.0.0" }