Merge pull request #1420 from PineappleIOnic/orchestration-api-test

Implement Utopia Orchestration library into Appwrite

CHANGES.md

@@ -1,9 +1,15 @@
+# Version 0.10.0 (Not Released Yet)
+- Switch from using Docker CLI to Docker API by intergrating [utopia-php/orchestration](https://github.com/utopia-php/orchestration)
+- Added DOCKERHUB_PULL_USERNAME, DOCKERHUB_PULL_PASSWORD and DOCKERHUB_PULL_EMAIL env variables for pulling from
+  private DockerHub repos
+
 # Version 0.9.4
 
 ## Security
 
 - Fixed security vulnerability that exposes project ID's from other admin users (#1453)
 
+
 # Version 0.9.3
 
 ## Bugs

app/config/variables.php

@@ -462,6 +462,33 @@ return [
                 'question' => '',
                 'filter' => ''
             ],
+            [
+                'name' => 'DOCKERHUB_PULL_USERNAME',
+                'description' => 'The username for hub.docker.com. This variable is used to pull images from hub.docker.com.',
+                'introduction' => '0.10.0',
+                'default' => '',
+                'required' => false,
+                'question' => '',
+                'filter' => ''
+            ],
+            [
+                'name' => 'DOCKERHUB_PULL_PASSWORD',
+                'description' => 'The password for hub.docker.com. This variable is used to pull images from hub.docker.com.',
+                'introduction' => '0.10.0',
+                'default' => '',
+                'required' => false,
+                'question' => '',
+                'filter' => ''
+            ],
+            [
+                'name' => 'DOCKERHUB_PULL_EMAIL',
+                'description' => 'The email for hub.docker.com. This variable is used to pull images from hub.docker.com.',
+                'introduction' => '0.10.0',
+                'default' => '',
+                'required' => false,
+                'question' => '',
+                'filter' => ''
+            ],
         ],
         [
             'category' => 'Maintenance',

app/workers/functions.php

@@ -13,6 +13,11 @@ use Swoole\Runtime;
 use Utopia\App;
 use Utopia\CLI\Console;
 use Utopia\Config\Config;
+use Utopia\Orchestration\Orchestration;
+use Utopia\Orchestration\Adapter\DockerAPI;
+use Utopia\Orchestration\Container;
+use Utopia\Orchestration\Exception\Orchestration as OrchestrationException;
+use Utopia\Orchestration\Exception\Timeout as TimeoutException;
 
 require_once __DIR__.'/../workers.php';
 
@@ -23,39 +28,27 @@ Console::success(APP_NAME.' functions worker v1 has started');
 
 $runtimes = Config::getParam('runtimes');
 
+$dockerUser = App::getEnv('DOCKERHUB_PULL_USERNAME', null);
+$dockerPass = App::getEnv('DOCKERHUB_PULL_PASSWORD', null);
+$dockerEmail = App::getEnv('DOCKERHUB_PULL_EMAIL', null);
+$orchestration = new Orchestration(new DockerAPI($dockerUser, $dockerPass, $dockerEmail));
+
 /**
  * Warmup Docker Images
  */
 $warmupStart = \microtime(true);
 
-Co\run(function() use ($runtimes) {  // Warmup: make sure images are ready to run fast 🚀
+Co\run(function() use ($runtimes, $orchestration) {  // Warmup: make sure images are ready to run fast 🚀
-
-    $dockerUser = App::getEnv('DOCKERHUB_PULL_USERNAME', null);
-    $dockerPass = App::getEnv('DOCKERHUB_PULL_PASSWORD', null);
-
-    if($dockerUser) {
-        $stdout = '';
-        $stderr = '';
-
-        Console::execute('docker login --username '.$dockerUser.' --password-stdin', $dockerPass, $stdout, $stderr);
-        Console::log('Docker Login'. $stdout.$stderr);
-    }
-
     foreach($runtimes as $runtime) {
-        go(function() use ($runtime) {
+        go(function() use ($runtime, $orchestration) {
-            $stdout = '';
-            $stderr = '';
-        
             Console::info('Warming up '.$runtime['name'].' '.$runtime['version'].' environment...');
-        
+                
-            Console::execute('docker pull '.$runtime['image'], '', $stdout, $stderr);
+            $response = $orchestration->pull($runtime['image']);
-        
+
-            if(!empty($stdout)) {
+            if ($response) {
-                Console::log($stdout);
+                Console::success("Successfully Warmed up {$runtime['name']} {$runtime['version']}!");
-            }
+            } else {
-        
+                Console::error("Failed to Warmup {$runtime['name']} {$runtime['version']}!");
-            if(!empty($stderr)) {
-                Console::error($stderr);
             }
         });
     }
@@ -74,40 +67,17 @@ $stderr = '';
 
 $executionStart = \microtime(true);
 
-$exitCode = Console::execute('docker ps --all --format "name={{.Names}}&status={{.Status}}&labels={{.Labels}}" --filter label=appwrite-type=function'
+$response = $orchestration->list(['label' => 'appwrite-type=function']);
-    , '', $stdout, $stderr, 30);
+
+$list = [];
+
+foreach ($response as $value) {
+    $list[$value->getName()] = $value;
+}
 
 $executionEnd = \microtime(true);
 
-$list = [];
+Console::info(count($list).' functions listed in ' . ($executionEnd - $executionStart) . ' seconds');
-$stdout = \explode("\n", $stdout);
-
-\array_map(function($value) use (&$list) {
-    $container = [];
-
-    \parse_str($value, $container);
-
-    if(isset($container['name'])) {
-        $container = [
-            'name' => $container['name'],
-            'online' => (\substr($container['status'], 0, 2) === 'Up'),
-            'status' => $container['status'],
-            'labels' => $container['labels'],
-        ];
-
-        \array_map(function($value) use (&$container) {
-            $value = \explode('=', $value);
-            
-            if(isset($value[0]) && isset($value[1])) {
-                $container[$value[0]] = $value[1];
-            }
-        }, \explode(',', $container['labels']));
-
-        $list[$container['name']] = $container;
-    }
-}, $stdout);
-
-Console::info(count($list)." functions listed in " . ($executionEnd - $executionStart) . " seconds with exit code {$exitCode}");
 
 /**
  * 1. Get event args - DONE
@@ -297,6 +267,7 @@ class FunctionsV1 extends Worker
     public function execute(string $trigger, string $projectId, string $executionId, Database $database, Document $function, string $event = '', string $eventData = '', string $data = '', array $webhooks = [], string $userId = '', string $jwt = ''): void
     {
         global $list;
+        global $orchestration;
 
         $runtimes = Config::getParam('runtimes');
 
@@ -355,12 +326,6 @@ class FunctionsV1 extends Worker
             'APPWRITE_FUNCTION_PROJECT_ID' => $projectId,
         ]);
 
-        \array_walk($vars, function (&$value, $key) {
-            $key = $this->filterEnvKey($key);
-            $value = \escapeshellarg((empty($value)) ? '' : $value);
-            $value = "--env {$key}={$value}";
-        });
-
         $tagPath = $tag->getAttribute('path', '');
         $tagPathTarget = '/tmp/project-'.$projectId.'/'.$tag->getId().'/code.tar.gz';
         $tagPathTargetDir = \pathinfo($tagPathTarget, PATHINFO_DIRNAME);
@@ -383,12 +348,14 @@ class FunctionsV1 extends Worker
             }
         }
 
-        if(isset($list[$container]) && !$list[$container]['online']) { // Remove conatiner if not online
+        if(isset($list[$container]) && !(\substr($list[$container]->getStatus(), 0, 2) === 'Up')) { // Remove conatiner if not online
             $stdout = '';
             $stderr = '';
-            
+
-            if(Console::execute("docker rm {$container}", '', $stdout, $stderr, 30) !== 0) {
+            try {
-                throw new Exception('Failed to remove offline container: '.$stderr);
+                $orchestration->remove($container);
+            } catch (Exception $e) {
+                Console::warning('Failed to remove container: '.$e->getMessage());
             }
 
             unset($list[$container]);
@@ -409,79 +376,102 @@ class FunctionsV1 extends Worker
     
             $executionStart = \microtime(true);
             $executionTime = \time();
-            $cpus = App::getEnv('_APP_FUNCTIONS_CPUS', '');
-            $memory = App::getEnv('_APP_FUNCTIONS_MEMORY', '');
-            $swap = App::getEnv('_APP_FUNCTIONS_MEMORY_SWAP', '');
-            $exitCode = Console::execute("docker run ".
-                " -d".
-                " --entrypoint=\"\"".
-                (empty($cpus) ? "" : (" --cpus=".$cpus)).
-                (empty($memory) ? "" : (" --memory=".$memory."m")).
-                (empty($swap) ? "" : (" --memory-swap=".$swap."m")).
-                " --name={$container}".
-                " --label appwrite-type=function".
-                " --label appwrite-created={$executionTime}".
-                " --volume {$tagPathTargetDir}:/tmp:rw".
-                " --workdir /usr/local/src".
-                " ".\implode(" ", $vars).
-                " {$runtime['image']}".
-                " tail -f /dev/null"
-            , '', $stdout, $stderr, 30);
 
-            if($exitCode !== 0) {
+            $orchestration->setCpus(App::getEnv('_APP_FUNCTIONS_CPUS', '1'));
-                throw new Exception('Failed to create function environment: '.$stderr);
+            $orchestration->setMemory(App::getEnv('_APP_FUNCTIONS_MEMORY', '256'));
+            $orchestration->setSwap(App::getEnv('_APP_FUNCTIONS_MEMORY_SWAP', '256'));
+
+            foreach($vars as &$value) {
+                $value = strval($value);
             }
 
-            $exitCodeUntar = Console::execute("docker exec ".
+            $id = $orchestration->run(
-                $container.
+                image: $runtime['image'],
-                " sh -c 'mv /tmp/code.tar.gz /usr/local/src/code.tar.gz && tar -zxf /usr/local/src/code.tar.gz --strip 1 && rm /usr/local/src/code.tar.gz'"
+                name: $container,
-                , '', $stdout, $stderr, 60);
+                command: ['tail',
+                '-f',
+                '/dev/null'
+                ],
+                entrypoint: '',
+                workdir: '/usr/local/src',
+                volumes: [],
+                vars: $vars,
+                mountFolder: $tagPathTargetDir,
+                labels: [
+                    'appwrite-type' => 'function',
+                    'appwrite-created' => strval($executionTime)
+                ]);
 
-            if($exitCodeUntar !== 0) {
+            $untarStdout = '';
-                throw new Exception('Failed to extract tar: '.$stderr);
+            $untarStderr = '';
+
+            $untarSuccess = $orchestration->execute(
+                name: $container, 
+                command: [
+                    'sh',
+                    '-c',
+                    'mv /tmp/code.tar.gz /usr/local/src/code.tar.gz && tar -zxf /usr/local/src/code.tar.gz --strip 1 && rm /usr/local/src/code.tar.gz'
+                ],
+                stdout: $untarStdout, 
+                stderr: $untarStderr,
+                vars: $vars,
+                timeout: 60);
+
+            if (!$untarSuccess) {
+                throw new Exception('Failed to extract tar: '.$untarStderr);
             }
 
             $executionEnd = \microtime(true);
 
-            $list[$container] = [
+            $list[$container] = new Container($container, $id, 'Up', 
-                'name' => $container,
+                [
-                'online' => true,
-                'status' => 'Up',
-                'labels' => [
                     'appwrite-type' => 'function',
-                    'appwrite-created' => $executionTime,
+                    'appwrite-created' => strval($executionTime),
-                ],
+                ]);
-            ];
 
-            Console::info("Function created in " . ($executionEnd - $executionStart) . " seconds with exit code {$exitCode}");
+            Console::info('Function created in ' . ($executionEnd - $executionStart) . ' seconds');
         }
         else {
             Console::info('Container is ready to run');
         }
-        
+
         $stdout = '';
         $stderr = '';
 
         $executionStart = \microtime(true);
-        
+
-        $exitCode = Console::execute("docker exec ".\implode(" ", $vars)." {$container} {$command}"
+        $exitCode = 0;
-            , '', $stdout, $stderr, $function->getAttribute('timeout', (int) App::getEnv('_APP_FUNCTIONS_TIMEOUT', 900)));
+
+        try {
+            $exitCode = (int)!$orchestration->execute(
+                name: $container, 
+                command: $orchestration->parseCommandString($command), 
+                stdout: $stdout, 
+                stderr: $stderr, 
+                vars: $vars, 
+                timeout: $function->getAttribute('timeout', (int) App::getEnv('_APP_FUNCTIONS_TIMEOUT', 900)));
+        } catch (TimeoutException $e) {
+            $exitCode = 124;
+        } catch (OrchestrationException $e) {
+            $stderr = $e->getMessage();
+            $exitCode = 1;
+        }
 
         $executionEnd = \microtime(true);
         $executionTime = ($executionEnd - $executionStart);
         $functionStatus = ($exitCode === 0) ? 'completed' : 'failed';
 
-        Console::info("Function executed in " . ($executionEnd - $executionStart) . " seconds with exit code {$exitCode}");
+        Console::info('Function executed in ' . ($executionEnd - $executionStart) . ' seconds, status: ' . $functionStatus);
 
         Authorization::disable();
-        
+
         $execution = $database->updateDocument(array_merge($execution->getArrayCopy(), [
             'tagId' => $tag->getId(),
             'status' => $functionStatus,
             'exitCode' => $exitCode,
             'stdout' => \mb_substr($stdout, -4000), // log last 4000 chars output
             'stderr' => \mb_substr($stderr, -4000), // log last 4000 chars output
-            'time' => $executionTime,
+            'time' => $executionTime
         ]));
         
         Authorization::reset();
@@ -529,6 +519,7 @@ class FunctionsV1 extends Worker
     public function cleanup(): void
     {
         global $list;
+        global $orchestration;
 
         Console::success(count($list).' running containers counted');
 
@@ -538,19 +529,17 @@ class FunctionsV1 extends Worker
             Console::info('Starting containers cleanup');
 
             \uasort($list, function ($item1, $item2) {
-                return (int)($item1['appwrite-created'] ?? 0) <=> (int)($item2['appwrite-created'] ?? 0);
+                return (int)($item1->getLabels['appwrite-created'] ?? 0) <=> (int)($item2->getLabels['appwrite-created'] ?? 0);
             });
 
             while(\count($list) > $max) {
                 $first = \array_shift($list);
-                $stdout = '';
-                $stderr = '';
 
-                if(Console::execute("docker rm -f {$first['name']}", '', $stdout, $stderr, 30) !== 0) {
+                try {
-                    Console::error('Failed to remove container: '.$stderr);
+                    $orchestration->remove($first->getName(), true);
-                }
+                    Console::info('Removed container: '.$first->getName());
-                else {
+                } catch (Exception $e) {
-                    Console::info('Removed container: '.$first['name']);
+                    Console::error('Failed to remove container: '.$e);
                 }
             }
         }

composer.json

@@ -54,6 +54,7 @@
         "utopia-php/image": "0.5.*",
         "resque/php-resque": "1.3.6",
         "matomo/device-detector": "4.2.3",
+        "utopia-php/orchestration": "0.2.0",
         "dragonmantank/cron-expression": "3.1.0",
         "influxdb/influxdb-php": "1.15.2",
         "phpmailer/phpmailer": "6.5.0",

composer.lock

@@ -4,7 +4,7 @@
         "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
         "This file is @generated automatically"
     ],
-    "content-hash": "45963af754680568d89330a4f37c40d1",
+    "content-hash": "0a782184d016e458ff18f20a2c49ccfb",
     "packages": [
         {
             "name": "adhocore/jwt",
@@ -1907,6 +1907,61 @@
             },
             "time": "2021-07-24T11:35:55+00:00"
         },
+        {
+            "name": "utopia-php/orchestration",
+            "version": "0.2.0",
+            "source": {
+                "type": "git",
+                "url": "https://github.com/utopia-php/orchestration.git",
+                "reference": "de10509017768cf2b62363bb39912002ab41dafb"
+            },
+            "dist": {
+                "type": "zip",
+                "url": "https://api.github.com/repos/utopia-php/orchestration/zipball/de10509017768cf2b62363bb39912002ab41dafb",
+                "reference": "de10509017768cf2b62363bb39912002ab41dafb",
+                "shasum": ""
+            },
+            "require": {
+                "php": ">=8.0",
+                "utopia-php/cli": "0.11.*"
+            },
+            "require-dev": {
+                "phpunit/phpunit": "^9.3",
+                "vimeo/psalm": "4.0.1"
+            },
+            "type": "library",
+            "autoload": {
+                "psr-4": {
+                    "Utopia\\Orchestration\\": "src/Orchestration"
+                }
+            },
+            "notification-url": "https://packagist.org/downloads/",
+            "license": [
+                "MIT"
+            ],
+            "authors": [
+                {
+                    "name": "Eldad Fux",
+                    "email": "eldad@appwrite.io"
+                }
+            ],
+            "description": "Lite & fast micro PHP abstraction library for container orchestration",
+            "keywords": [
+                "docker",
+                "framework",
+                "kubernetes",
+                "orchestration",
+                "php",
+                "swarm",
+                "upf",
+                "utopia"
+            ],
+            "support": {
+                "issues": "https://github.com/utopia-php/orchestration/issues",
+                "source": "https://github.com/utopia-php/orchestration/tree/0.2.0"
+            },
+            "time": "2021-08-16T12:52:42+00:00"
+        },
         {
             "name": "utopia-php/preloader",
             "version": "0.2.4",
@@ -2452,16 +2507,16 @@
         },
         {
             "name": "composer/package-versions-deprecated",
-            "version": "1.11.99.2",
+            "version": "1.11.99.3",
             "source": {
                 "type": "git",
                 "url": "https://github.com/composer/package-versions-deprecated.git",
-                "reference": "c6522afe5540d5fc46675043d3ed5a45a740b27c"
+                "reference": "fff576ac850c045158a250e7e27666e146e78d18"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/composer/package-versions-deprecated/zipball/c6522afe5540d5fc46675043d3ed5a45a740b27c",
+                "url": "https://api.github.com/repos/composer/package-versions-deprecated/zipball/fff576ac850c045158a250e7e27666e146e78d18",
-                "reference": "c6522afe5540d5fc46675043d3ed5a45a740b27c",
+                "reference": "fff576ac850c045158a250e7e27666e146e78d18",
                 "shasum": ""
             },
             "require": {
@@ -2505,7 +2560,7 @@
             "description": "Composer plugin that provides efficient querying for installed package versions (no runtime IO)",
             "support": {
                 "issues": "https://github.com/composer/package-versions-deprecated/issues",
-                "source": "https://github.com/composer/package-versions-deprecated/tree/1.11.99.2"
+                "source": "https://github.com/composer/package-versions-deprecated/tree/1.11.99.3"
             },
             "funding": [
                 {
@@ -2521,7 +2576,7 @@
                     "type": "tidelift"
                 }
             ],
-            "time": "2021-05-24T07:46:03+00:00"
+            "time": "2021-08-17T13:49:14+00:00"
         },
         {
             "name": "composer/semver",
@@ -5047,16 +5102,16 @@
         },
         {
             "name": "symfony/console",
-            "version": "v5.3.6",
+            "version": "v5.3.7",
             "source": {
                 "type": "git",
                 "url": "https://github.com/symfony/console.git",
-                "reference": "51b71afd6d2dc8f5063199357b9880cea8d8bfe2"
+                "reference": "8b1008344647462ae6ec57559da166c2bfa5e16a"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/symfony/console/zipball/51b71afd6d2dc8f5063199357b9880cea8d8bfe2",
+                "url": "https://api.github.com/repos/symfony/console/zipball/8b1008344647462ae6ec57559da166c2bfa5e16a",
-                "reference": "51b71afd6d2dc8f5063199357b9880cea8d8bfe2",
+                "reference": "8b1008344647462ae6ec57559da166c2bfa5e16a",
                 "shasum": ""
             },
             "require": {
@@ -5126,7 +5181,7 @@
                 "terminal"
             ],
             "support": {
-                "source": "https://github.com/symfony/console/tree/v5.3.6"
+                "source": "https://github.com/symfony/console/tree/v5.3.7"
             },
             "funding": [
                 {
@@ -5142,7 +5197,7 @@
                     "type": "tidelift"
                 }
             ],
-            "time": "2021-07-27T19:10:22+00:00"
+            "time": "2021-08-25T20:02:16+00:00"
         },
         {
             "name": "symfony/deprecation-contracts",
@@ -5699,16 +5754,16 @@
         },
         {
             "name": "symfony/string",
-            "version": "v5.3.3",
+            "version": "v5.3.7",
             "source": {
                 "type": "git",
                 "url": "https://github.com/symfony/string.git",
-                "reference": "bd53358e3eccec6a670b5f33ab680d8dbe1d4ae1"
+                "reference": "8d224396e28d30f81969f083a58763b8b9ceb0a5"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/symfony/string/zipball/bd53358e3eccec6a670b5f33ab680d8dbe1d4ae1",
+                "url": "https://api.github.com/repos/symfony/string/zipball/8d224396e28d30f81969f083a58763b8b9ceb0a5",
-                "reference": "bd53358e3eccec6a670b5f33ab680d8dbe1d4ae1",
+                "reference": "8d224396e28d30f81969f083a58763b8b9ceb0a5",
                 "shasum": ""
             },
             "require": {
@@ -5762,7 +5817,7 @@
                 "utf8"
             ],
             "support": {
-                "source": "https://github.com/symfony/string/tree/v5.3.3"
+                "source": "https://github.com/symfony/string/tree/v5.3.7"
             },
             "funding": [
                 {
@@ -5778,7 +5833,7 @@
                     "type": "tidelift"
                 }
             ],
-            "time": "2021-06-27T11:44:38+00:00"
+            "time": "2021-08-26T08:00:08+00:00"
         },
         {
             "name": "theseer/tokenizer",
@@ -6089,5 +6144,5 @@
     "platform-overrides": {
         "php": "8.0"
     },
-    "plugin-api-version": "2.1.0"
+    "plugin-api-version": "2.0.0"
 }

tests/e2e/Services/Functions/FunctionsCustomClientTest.php

@@ -196,7 +196,6 @@ class FunctionsCustomClientTest extends Scope
         ]);
 
         $output = json_decode($executions['body']['stdout'], true);
-
         $this->assertEquals(200, $executions['headers']['status-code']);
         $this->assertEquals('completed', $executions['body']['status']);
         $this->assertEquals($functionId, $output['APPWRITE_FUNCTION_ID']);

tests/e2e/Services/Functions/FunctionsCustomServerTest.php

@@ -518,7 +518,7 @@ class FunctionsCustomServerTest extends Scope
         $this->assertEquals($executions['body']['executions'][0]['$id'], $executionId);
         $this->assertEquals($executions['body']['executions'][0]['trigger'], 'http');
         $this->assertEquals($executions['body']['executions'][0]['status'], 'failed');
-        $this->assertEquals($executions['body']['executions'][0]['exitCode'], 1);
+        $this->assertEquals($executions['body']['executions'][0]['exitCode'], 124);
         $this->assertGreaterThan(2, $executions['body']['executions'][0]['time']);
         $this->assertLessThan(3, $executions['body']['executions'][0]['time']);
         $this->assertEquals($executions['body']['executions'][0]['stdout'], '');