diff --git a/.env b/.env index 3f8e511b6..f59620ec5 100644 --- a/.env +++ b/.env @@ -39,7 +39,7 @@ _APP_FUNCTIONS_CONTAINERS=10 _APP_FUNCTIONS_CPUS=4 _APP_FUNCTIONS_MEMORY=2000 _APP_FUNCTIONS_MEMORY_SWAP=2000 -_APP_EXECUTOR_SECRET=a-randomly-generated-key +_APP_EXECUTOR_SECRET=a-random-secret _APP_MAINTENANCE_INTERVAL=86400 _APP_MAINTENANCE_RETENTION_EXECUTION=1209600 _APP_MAINTENANCE_RETENTION_ABUSE=86400 diff --git a/Dockerfile b/Dockerfile index a9f1b0fb2..167d528a6 100755 --- a/Dockerfile +++ b/Dockerfile @@ -269,6 +269,7 @@ RUN chmod +x /usr/local/bin/doctor && \ chmod +x /usr/local/bin/worker-database && \ chmod +x /usr/local/bin/worker-deletes && \ chmod +x /usr/local/bin/worker-functions && \ + chmod +x /usr/local/bin/worker-builds && \ chmod +x /usr/local/bin/worker-mails && \ chmod +x /usr/local/bin/worker-webhooks diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 2454551ae..c418bc72f 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -566,6 +566,8 @@ App::post('/v1/functions/:functionId/tags') } $tagId = $dbForProject->getId(); + + // TODO : What should be the read and write permissons of a tag? $tag = $dbForProject->createDocument('tags', new Document([ '$id' => $tagId, '$read' => ['role:all'], @@ -586,42 +588,14 @@ App::post('/v1/functions/:functionId/tags') ->setParam('storage', $tag->getAttribute('size', 0)) ; - // Send start build reqeust to executor using /v1/tag - $function = $dbForProject->getDocument('functions', $functionId); - - $ch = \curl_init(); - \curl_setopt($ch, CURLOPT_URL, "http://appwrite-executor:8080/v1/tag"); - \curl_setopt($ch, CURLOPT_POST, true); - \curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode([ + // Enqueue a message to start the build + Resque::enqueue('v1-builds', 'BuildsV1', [ + 'projectId' => $project->getId(), 'functionId' => $function->getId(), - 'tagId' => $tag->getId(), - 'userId' => $user->getId(), - ])); - \curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); - \curl_setopt($ch, CURLOPT_TIMEOUT, 900); - \curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 10); - \curl_setopt($ch, CURLOPT_HTTPHEADER, [ - 'Content-Type: application/json', - 'x-appwrite-project: '.$project->getId(), - 'x-appwrite-executor-key: '. App::getEnv('_APP_EXECUTOR_SECRET', '') + 'tagId' => $tagId, + 'type' => 'tag' ]); - $executorResponse = \curl_exec($ch); - - $error = \curl_error($ch); - - if (!empty($error)) { - throw new Exception('Executor Communication Error: ' . $error, 500); - } - - // Check status code - $statusCode = \curl_getinfo($ch, CURLINFO_HTTP_CODE); - if (200 !== $statusCode) { - throw new Exception('Executor error: ' . $executorResponse, $statusCode); - } - - \curl_close($ch); - $response->setStatusCode(Response::STATUS_CODE_CREATED); $response->dynamic($tag, Response::MODEL_TAG); }); diff --git a/app/executor.php b/app/executor.php index 0d3365fb5..aed491cfa 100644 --- a/app/executor.php +++ b/app/executor.php @@ -151,15 +151,22 @@ function createRuntimeServer(string $functionId, string $projectId, string $tagI global $runtimes; global $activeFunctions; + var_dump("Im here 1"); + try { $orchestration = $orchestrationPool->get(); + var_dump("Im here 2"); $function = $database->getDocument('functions', $functionId); + var_dump("Im here 3"); $tag = $database->getDocument('tags', $tagId); + var_dump("Im here 4"); if ($tag->getAttribute('buildId') === null) { throw new Exception('Tag has no buildId'); } + var_dump("Im here 5"); + // Grab Build Document $build = $database->getDocument('builds', $tag->getAttribute('buildId')); @@ -170,6 +177,8 @@ function createRuntimeServer(string $functionId, string $projectId, string $tagI return; } + var_dump("Im here"); + // Generate random secret key $secret = \bin2hex(\random_bytes(16)); @@ -787,122 +796,27 @@ App::post('/v1/cleanup/tag') return $response->json(['success' => true]); }); -App::post('/v1/tag') + +App::post('/v1/create/runtime') + ->desc('Create a new runtime server') ->param('functionId', '', new UID(), 'Function unique ID.') ->param('tagId', '', new UID(), 'Tag unique ID.') - ->param('userId', '', new UID(), 'User unique ID.', true) + ->inject('projectID') ->inject('response') ->inject('dbForProject') - ->inject('projectID') - ->inject('register') - ->action(function (string $functionId, string $tagId, string $userId, Response $response, Database $dbForProject, string $projectID, Registry $register) use ($runtimes) { - // Get function document - $function = $dbForProject->getDocument('functions', $functionId); - // Get tag document - $tag = $dbForProject->getDocument('tags', $tagId); - - // Check if both documents exist - if ($function->isEmpty()) { - throw new Exception('Function not found', 404); - } - - if ($tag->isEmpty()) { - throw new Exception('Tag not found', 404); - } - - $runtime = $runtimes[$function->getAttribute('runtime')] ?? null; - - if (\is_null($runtime)) { - throw new Exception('Runtime "' . $function->getAttribute('runtime', '') . '" is not supported'); - } - - // Create a new build entry - $buildId = $dbForProject->getId(); - - if ($tag->getAttribute('buildId')) { - $buildId = $tag->getAttribute('buildId'); - } else { - try { - $dbForProject->createDocument('builds', new Document([ - '$id' => $buildId, - '$read' => (!empty($userId)) ? ['user:' . $userId] : [], - '$write' => ['role:all'], - 'dateCreated' => time(), - 'status' => 'processing', - 'runtime' => $function->getAttribute('runtime'), - 'outputPath' => '', - 'source' => $tag->getAttribute('path'), - 'sourceType' => Storage::DEVICE_LOCAL, - 'stdout' => '', - 'stderr' => '', - 'buildTime' => 0, - 'envVars' => [ - 'ENTRYPOINT_NAME' => $tag->getAttribute('entrypoint'), - 'APPWRITE_FUNCTION_ID' => $function->getId(), - 'APPWRITE_FUNCTION_NAME' => $function->getAttribute('name', ''), - 'APPWRITE_FUNCTION_RUNTIME_NAME' => $runtime['name'], - 'APPWRITE_FUNCTION_RUNTIME_VERSION' => $runtime['version'], - 'APPWRITE_FUNCTION_PROJECT_ID' => $projectID, - ] - ])); - - $tag->setAttribute('buildId', $buildId); - - $dbForProject->updateDocument('tags', $tag->getId(), $tag); - } catch (\Throwable $th) { - var_dump($tag->getArrayCopy()); - throw $th; - } - } - - // Build Code - go(function () use ($projectID, $tagId, $buildId, $functionId, $function, $register) { - $db = $register->get('dbPool')->get(); - $redis = $register->get('redisPool')->get(); - $cache = new Cache(new RedisCache($redis)); - - $dbForProject = new Database(new MariaDB($db), $cache); - $dbForProject->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); - $dbForProject->setNamespace('_project_' . $projectID); - // Build Code - runBuildStage($buildId, $projectID); - - // Update the schedule - $schedule = $function->getAttribute('schedule', ''); - $cron = (empty($function->getAttribute('tag')) && !empty($schedule)) ? new CronExpression($schedule) : null; - $next = (empty($function->getAttribute('tag')) && !empty($schedule)) ? $cron->getNextRunDate()->format('U') : 0; - - // Grab tag - $tag = $dbForProject->getDocument('tags', $tagId); - - // Grab build - $build = $dbForProject->getDocument('builds', $buildId); - - // If the build failed, it won't be possible to deploy - if ($build->getAttribute('status') !== 'ready') { - return; - } - - if ($tag->getAttribute('automaticDeploy') === true) { - // Update the function document setting the tag as the active one - $function - ->setAttribute('tag', $tag->getId()) - ->setAttribute('scheduleNext', (int)$next); - $function = $dbForProject->updateDocument('functions', $function->getId(), $function); - } - - // Deploy Runtime Server + ->action(function (string $functionId, string $tagId, string $projectID, Response $response, Database $dbForProject) { + try { + Console::success('Creating runtime for tag ' . $tagId); createRuntimeServer($functionId, $projectID, $tagId, $dbForProject); + } catch (\Throwable $th) { + $response + ->setStatusCode(400) + ->json(['error' => $th->getMessage()]); + }; - $register->get('dbPool')->put($db); - $register->get('redisPool')->put($redis); - }); - - if (false === $function) { - throw new Exception('Failed saving function to DB', 500); - } - - $response->dynamic($function, Response::MODEL_FUNCTION); + $response + ->setStatusCode(201) + ->noContent(); }); App::get('/v1/') @@ -943,10 +857,8 @@ App::post('/v1/build/:buildId') // Start a Build throw new Exception('Build is already finished', 409); } - go(function () use ($buildId, $dbForProject, $projectID) { - // Build Code - runBuildStage($buildId, $projectID, $dbForProject); - }); + // Build Code + runBuildStage($buildId, $projectID, $dbForProject); // return success return $response->json(['success' => true]); diff --git a/app/workers/builds.php b/app/workers/builds.php new file mode 100644 index 000000000..e98a8e7fc --- /dev/null +++ b/app/workers/builds.php @@ -0,0 +1,253 @@ +args['type'] ?? ''; + $projectId = $this->args['projectId'] ?? ''; + + switch ($type) { + case 'tag': + $functionId = $this->args['functionId'] ?? ''; + $tagId = $this->args['tagId'] ?? ''; + Console::success("Creating build for tag: $tagId"); + $this->buildTag($projectId, $functionId, $tagId); + break; + default: + throw new \Exception('Invalid trigger'); + break; + } + } + + protected function triggerBuildStage(string $projectId, string $buildId) + { + // TODO: What is a reasonable time to wait for a build to complete? + $ch = \curl_init(); + \curl_setopt($ch, CURLOPT_URL, "http://appwrite-executor:8080/v1/build/$buildId"); + \curl_setopt($ch, CURLOPT_POST, true); + \curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); + \curl_setopt($ch, CURLOPT_TIMEOUT, 900); + \curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 10); + \curl_setopt($ch, CURLOPT_HTTPHEADER, [ + 'Content-Type: application/json', + 'x-appwrite-project: '.$projectId, + 'x-appwrite-executor-key: '. App::getEnv('_APP_EXECUTOR_SECRET', '') + ]); + + $response = \curl_exec($ch); + $responseStatus = curl_getinfo($ch, CURLINFO_HTTP_CODE); + + $error = \curl_error($ch); + if (!empty($error)) { + throw new \Exception($error); + } + + \curl_close($ch); + + if ($responseStatus !== 200) { + throw new \Exception("Build failed with status code: $responseStatus"); + } + + $response = json_decode($response, true); + if (isset($response['error'])) { + throw new \Exception($response['error']); + } + + if (isset($response['success']) && $response['success'] === true) { + return; + } else { + throw new \Exception("Build failed"); + } + } + + protected function triggerCreateRuntimeServer(string $projectId, string $functionId, string $tagId) + { + $ch = \curl_init(); + \curl_setopt($ch, CURLOPT_URL, "http://appwrite-executor:8080/v1/create/runtime"); + \curl_setopt($ch, CURLOPT_POST, true); + \curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); + \curl_setopt($ch, CURLOPT_TIMEOUT, 900); + \curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 10); + \curl_setopt($ch, CURLOPT_HTTPHEADER, [ + 'Content-Type: application/json', + 'x-appwrite-project: '.$projectId, + 'x-appwrite-executor-key: '. App::getEnv('_APP_EXECUTOR_SECRET', '') + ]); + \curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode([ + 'functionId' => $functionId, + 'tagId' => $tagId + ])); + + $response = \curl_exec($ch); + $responseStatus = curl_getinfo($ch, CURLINFO_HTTP_CODE); + + $error = \curl_error($ch); + if (!empty($error)) { + throw new \Exception($error); + } + + \curl_close($ch); + + if ($responseStatus !== 200) { + throw new \Exception("Build failed with status code: $responseStatus"); + } + + $response = json_decode($response, true); + if (isset($response['error'])) { + throw new \Exception($response['error']); + } + + if (isset($response['success']) && $response['success'] === true) { + return; + } else { + throw new \Exception("Build failed"); + } + } + + protected function buildTag(string $projectId, string $functionId, string $tagId) + { + $dbForProject = $this->getProjectDB($projectId); + + // TODO: Why does it need to skip authorization? + $function = Authorization::skip(fn() => $dbForProject->getDocument('functions', $functionId)); + + if ($function->isEmpty()) { + throw new Exception('Function not found', 404); + } + + // Get tag document + $tag = $dbForProject->getDocument('tags', $tagId); + if ($tag->isEmpty()) { + throw new Exception('Tag not found', 404); + } + + $runtimes = Config::getParam('runtimes', []); + $key = $function->getAttribute('runtime'); + $runtime = isset($runtimes[$key]) ? $runtimes[$key] : null; + if (\is_null($runtime)) { + throw new Exception('Runtime "' . $function->getAttribute('runtime', '') . '" is not supported'); + } + + $buildId = $tag->getAttribute('buildId', ''); + + // If build ID is empty, create a new build + if (empty($buildId)) { + try { + $buildId = $dbForProject->getId(); + // TODO : There is no way to associate a build with a tag. So we need to add a tagId attribute to the build document + // TODO : What should be the read and write permissions for a build ? + $dbForProject->createDocument('builds', new Document([ + '$id' => $buildId, + '$read' => ['role:all'], + '$write' => ['role:all'], + 'dateCreated' => time(), + 'status' => 'processing', + 'runtime' => $function->getAttribute('runtime'), + 'outputPath' => '', + 'source' => $tag->getAttribute('path'), + 'sourceType' => Storage::DEVICE_LOCAL, + 'stdout' => '', + 'stderr' => '', + 'buildTime' => 0, + 'envVars' => [ + 'ENTRYPOINT_NAME' => $tag->getAttribute('entrypoint'), + 'APPWRITE_FUNCTION_ID' => $function->getId(), + 'APPWRITE_FUNCTION_NAME' => $function->getAttribute('name', ''), + 'APPWRITE_FUNCTION_RUNTIME_NAME' => $runtime['name'], + 'APPWRITE_FUNCTION_RUNTIME_VERSION' => $runtime['version'], + 'APPWRITE_FUNCTION_PROJECT_ID' => $projectId, + ] + ])); + + $tag->setAttribute('buildId', $buildId); + $tag = $dbForProject->updateDocument('tags', $tagId, $tag); + + } catch (\Throwable $th) { + Console::error($th->getMessage()); + $tag->setAttribute('status', 'failed'); + $tag->setAttribute('buildId', ''); + $tag = $dbForProject->updateDocument('tags', $tagId, $tag); + return; + } + } + + // Build the Code + try { + Console::success("Creating Build with id: $buildId"); + $tag->setAttribute('status', 'building'); + $tag = $dbForProject->updateDocument('tags', $tagId, $tag); + $this->triggerBuildStage($projectId, $buildId); + } catch (\Throwable $th) { + Console::error($th->getMessage()); + $tag->setAttribute('status', 'failed'); + $tag = $dbForProject->updateDocument('tags', $tagId, $tag); + return; + } + + Console::success("Build id: $buildId completed successfully"); + + // Update the schedule + $schedule = $function->getAttribute('schedule', ''); + $cron = (empty($function->getAttribute('tag')) && !empty($schedule)) ? new CronExpression($schedule) : null; + $next = (empty($function->getAttribute('tag')) && !empty($schedule)) ? $cron->getNextRunDate()->format('U') : 0; + + // Grab build + $build = $dbForProject->getDocument('builds', $buildId); + + // If the build failed, it won't be possible to deploy + if ($build->getAttribute('status') !== 'ready') { + throw new Exception('Build failed', 500); + } + + if ($tag->getAttribute('automaticDeploy') === true) { + // Update the function document setting the tag as the active one + $function + ->setAttribute('tag', $tag->getId()) + ->setAttribute('scheduleNext', (int)$next); + $function = $dbForProject->updateDocument('functions', $function->getId(), $function); + } + + // Deploy Runtime Server + try { + Console::success("Creating Runtime Server"); + $this->triggerCreateRuntimeServer($functionId, $projectId, $tagId, $dbForProject); + } catch (\Throwable $th) { + Console::error($th->getMessage()); + $tag->setAttribute('status', 'failed'); + $tag = $dbForProject->updateDocument('tags', $tagId, $tag); + return; + } + + Console::success("Runtime Server created successfully"); + } + + public function shutdown(): void + { + Console::success("Shutting Down..."); + } +} diff --git a/bin/worker-builds b/bin/worker-builds new file mode 100644 index 000000000..2ba26ef4d --- /dev/null +++ b/bin/worker-builds @@ -0,0 +1,10 @@ +#!/bin/sh + +if [ -z "$_APP_REDIS_USER" ] && [ -z "$_APP_REDIS_PASS" ] +then + REDIS_BACKEND="${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" +else + REDIS_BACKEND="redis://${_APP_REDIS_USER}:${_APP_REDIS_PASS}@${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" +fi + +INTERVAL=0.1 QUEUE='v1-builds' APP_INCLUDE='/usr/src/code/app/workers/builds.php' php /usr/src/code/vendor/bin/resque -dopcache.preload=opcache.preload=/usr/src/code/app/preload.php \ No newline at end of file diff --git a/composer.json b/composer.json index a85680d93..02daadc7d 100644 --- a/composer.json +++ b/composer.json @@ -50,7 +50,7 @@ "utopia-php/cache": "0.4.*", "utopia-php/cli": "0.11.*", "utopia-php/config": "0.2.*", - "utopia-php/database": "0.13.*", + "utopia-php/database": "0.14.*", "utopia-php/locale": "0.4.*", "utopia-php/registry": "0.5.*", "utopia-php/preloader": "0.2.*", diff --git a/composer.lock b/composer.lock index a85e49786..5d9a38a9b 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "cba39f50398d5ae2b121db34c9e4c529", + "content-hash": "1a5d84f96eb76e59f7ad0ff7bcd4a8d8", "packages": [ { "name": "adhocore/jwt", @@ -2135,16 +2135,16 @@ }, { "name": "utopia-php/database", - "version": "0.13.2", + "version": "0.14.0", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "bf92279b707b3a10ee5ec5df5c065023b2221357" + "reference": "2f2527bb080cf578fba327ea2ec637064561d403" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/bf92279b707b3a10ee5ec5df5c065023b2221357", - "reference": "bf92279b707b3a10ee5ec5df5c065023b2221357", + "url": "https://api.github.com/repos/utopia-php/database/zipball/2f2527bb080cf578fba327ea2ec637064561d403", + "reference": "2f2527bb080cf578fba327ea2ec637064561d403", "shasum": "" }, "require": { @@ -2192,9 +2192,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/0.13.2" + "source": "https://github.com/utopia-php/database/tree/0.14.0" }, - "time": "2022-01-04T10:51:22+00:00" + "time": "2022-01-21T16:34:34+00:00" }, { "name": "utopia-php/domains", @@ -3126,23 +3126,23 @@ }, { "name": "composer/pcre", - "version": "1.0.0", + "version": "1.0.1", "source": { "type": "git", "url": "https://github.com/composer/pcre.git", - "reference": "3d322d715c43a1ac36c7fe215fa59336265500f2" + "reference": "67a32d7d6f9f560b726ab25a061b38ff3a80c560" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/composer/pcre/zipball/3d322d715c43a1ac36c7fe215fa59336265500f2", - "reference": "3d322d715c43a1ac36c7fe215fa59336265500f2", + "url": "https://api.github.com/repos/composer/pcre/zipball/67a32d7d6f9f560b726ab25a061b38ff3a80c560", + "reference": "67a32d7d6f9f560b726ab25a061b38ff3a80c560", "shasum": "" }, "require": { "php": "^5.3.2 || ^7.0 || ^8.0" }, "require-dev": { - "phpstan/phpstan": "^1", + "phpstan/phpstan": "^1.3", "phpstan/phpstan-strict-rules": "^1.1", "symfony/phpunit-bridge": "^4.2 || ^5" }, @@ -3177,7 +3177,7 @@ ], "support": { "issues": "https://github.com/composer/pcre/issues", - "source": "https://github.com/composer/pcre/tree/1.0.0" + "source": "https://github.com/composer/pcre/tree/1.0.1" }, "funding": [ { @@ -3193,7 +3193,7 @@ "type": "tidelift" } ], - "time": "2021-12-06T15:17:27+00:00" + "time": "2022-01-21T20:24:37+00:00" }, { "name": "composer/semver", @@ -3697,9 +3697,6 @@ "require": { "php": "^7.1 || ^8.0" }, - "replace": { - "myclabs/deep-copy": "self.version" - }, "require-dev": { "doctrine/collections": "^1.0", "doctrine/common": "^2.6", @@ -6665,5 +6662,5 @@ "platform-overrides": { "php": "8.0" }, - "plugin-api-version": "2.1.0" + "plugin-api-version": "2.2.0" } diff --git a/docker-compose.yml b/docker-compose.yml index 97fe60c81..f38bfca05 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -286,6 +286,34 @@ services: - _APP_DB_PASS - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG + + appwrite-worker-builds: + entrypoint: worker-builds + container_name: appwrite-worker-builds + build: + context: . + networks: + - appwrite + volumes: + - ./app:/usr/src/code/app + - ./src:/usr/src/code/src + depends_on: + - redis + - mariadb + environment: + - _APP_ENV + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS + - _APP_LOGGING_PROVIDER + - _APP_LOGGING_CONFIG + - _APP_EXECUTOR_SECRET appwrite-worker-certificates: entrypoint: worker-certificates