From 074a293d633cbfa692097d40eb9a2cec8350c399 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Tue, 25 Jan 2022 00:06:29 +0400 Subject: [PATCH] feat: refactoring the builds worker --- app/executor.php | 15 ++++---------- app/http.php | 5 +++++ app/workers/builds.php | 44 +++++++++++++++++++----------------------- 3 files changed, 29 insertions(+), 35 deletions(-) diff --git a/app/executor.php b/app/executor.php index aed491cfa..93774dc2c 100644 --- a/app/executor.php +++ b/app/executor.php @@ -151,22 +151,15 @@ function createRuntimeServer(string $functionId, string $projectId, string $tagI global $runtimes; global $activeFunctions; - var_dump("Im here 1"); - try { $orchestration = $orchestrationPool->get(); - var_dump("Im here 2"); $function = $database->getDocument('functions', $functionId); - var_dump("Im here 3"); $tag = $database->getDocument('tags', $tagId); - var_dump("Im here 4"); if ($tag->getAttribute('buildId') === null) { throw new Exception('Tag has no buildId'); } - var_dump("Im here 5"); - // Grab Build Document $build = $database->getDocument('builds', $tag->getAttribute('buildId')); @@ -177,8 +170,6 @@ function createRuntimeServer(string $functionId, string $projectId, string $tagI return; } - var_dump("Im here"); - // Generate random secret key $secret = \bin2hex(\random_bytes(16)); @@ -318,7 +309,8 @@ function createRuntimeServer(string $functionId, string $projectId, string $tagI Console::info('Runtime server is ready to run'); } } catch (\Throwable $th) { - $orchestrationPool->put($orchestration); + var_dump($th->getTraceAsString()); + $orchestrationPool->put($orchestration ?? null); throw $th; } finally { $orchestrationPool->put($orchestration); @@ -797,7 +789,7 @@ App::post('/v1/cleanup/tag') }); -App::post('/v1/create/runtime') +App::post('/v1/executor/runtime') ->desc('Create a new runtime server') ->param('functionId', '', new UID(), 'Function unique ID.') ->param('tagId', '', new UID(), 'Tag unique ID.') @@ -857,6 +849,7 @@ App::post('/v1/build/:buildId') // Start a Build throw new Exception('Build is already finished', 409); } + Console::success('Starting build ' . $buildId); // Build Code runBuildStage($buildId, $projectID, $dbForProject); diff --git a/app/http.php b/app/http.php index a73daa256..3c6256d3b 100644 --- a/app/http.php +++ b/app/http.php @@ -21,6 +21,11 @@ use Utopia\Logger\Log\User; $http = new Server("0.0.0.0", App::getEnv('PORT', 80)); +$http->set([ + 'worker_num' => 1, + 'reactor_num' => 6 +]); + $payloadSize = max(4000000 /* 4mb */, App::getEnv('_APP_STORAGE_LIMIT', 10000000 /* 10mb */)); $http diff --git a/app/workers/builds.php b/app/workers/builds.php index d628d05be..7aec17e8f 100644 --- a/app/workers/builds.php +++ b/app/workers/builds.php @@ -11,9 +11,13 @@ use Utopia\Config\Config; require_once __DIR__.'/../init.php'; +// Disable Auth since we already validate it in the API +Authorization::disable(); + Console::title('Builds V1 Worker'); Console::success(APP_NAME.' build worker v1 has started'); +// TODO: Executor should return appropriate response codes. class BuildsV1 extends Worker { @@ -35,14 +39,14 @@ class BuildsV1 extends Worker case BUILD_TYPE_TAG: $functionId = $this->args['functionId'] ?? ''; $tagId = $this->args['tagId'] ?? ''; - Console::success("Creating build for tag: $tagId"); + Console::info("[ INFO ] Creating build for tag: $tagId"); $this->buildTag($projectId, $functionId, $tagId); break; case BUILD_TYPE_RETRY: $buildId = $this->args['buildId'] ?? ''; - Console::success("Retrying build for id: $buildId"); - $this->triggerBuildStage($projectId, $buildId,); + Console::info("[ INFO ] Retrying build for id: $buildId"); + $this->triggerBuild($projectId, $buildId); break; default: @@ -51,7 +55,7 @@ class BuildsV1 extends Worker } } - protected function triggerBuildStage(string $projectId, string $buildId) + protected function triggerBuild(string $projectId, string $buildId) { // TODO: What is a reasonable time to wait for a build to complete? $ch = \curl_init(); @@ -95,7 +99,7 @@ class BuildsV1 extends Worker protected function triggerCreateRuntimeServer(string $projectId, string $functionId, string $tagId) { $ch = \curl_init(); - \curl_setopt($ch, CURLOPT_URL, "http://appwrite-executor:8080/v1/create/runtime"); + \curl_setopt($ch, CURLOPT_URL, "http://appwrite-executor:8080/v1/executor/runtime"); \curl_setopt($ch, CURLOPT_POST, true); \curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); \curl_setopt($ch, CURLOPT_TIMEOUT, 900); @@ -120,7 +124,7 @@ class BuildsV1 extends Worker \curl_close($ch); - if ($responseStatus !== 200) { + if ($responseStatus >= 400) { throw new \Exception("Build failed with status code: $responseStatus"); } @@ -128,20 +132,13 @@ class BuildsV1 extends Worker if (isset($response['error'])) { throw new \Exception($response['error']); } - - if (isset($response['success']) && $response['success'] === true) { - return; - } else { - throw new \Exception("Build failed"); - } } protected function buildTag(string $projectId, string $functionId, string $tagId) { $dbForProject = $this->getProjectDB($projectId); - // TODO: Why does it need to skip authorization? - $function = Authorization::skip(fn() => $dbForProject->getDocument('functions', $functionId)); + $function = $dbForProject->getDocument('functions', $functionId); if ($function->isEmpty()) { throw new Exception('Function not found', 404); @@ -170,8 +167,8 @@ class BuildsV1 extends Worker // TODO : What should be the read and write permissions for a build ? $dbForProject->createDocument('builds', new Document([ '$id' => $buildId, - '$read' => ['role:all'], - '$write' => ['role:all'], + '$read' => [], + '$write' => [], 'dateCreated' => time(), 'status' => 'processing', 'runtime' => $function->getAttribute('runtime'), @@ -205,10 +202,10 @@ class BuildsV1 extends Worker // Build the Code try { - Console::success("Creating Build with id: $buildId"); + Console::info("[ INFO ] Creating build with id: $buildId"); $tag->setAttribute('status', 'building'); $tag = $dbForProject->updateDocument('tags', $tagId, $tag); - $this->triggerBuildStage($projectId, $buildId); + $this->triggerBuild($projectId, $buildId); } catch (\Throwable $th) { Console::error($th->getMessage()); $tag->setAttribute('status', 'failed'); @@ -216,7 +213,7 @@ class BuildsV1 extends Worker return; } - Console::success("Build id: $buildId completed successfully"); + Console::success("[ SUCCESS ] Build id: $buildId completed"); // Update the schedule $schedule = $function->getAttribute('schedule', ''); @@ -237,14 +234,13 @@ class BuildsV1 extends Worker ->setAttribute('tag', $tag->getId()) ->setAttribute('scheduleNext', (int)$next); - // TODO: Why should we disable auth ? - $function = Authorization::skip(fn() => $dbForProject->updateDocument('functions', $functionId, $function)); + $function = $dbForProject->updateDocument('functions', $functionId, $function); } // Deploy Runtime Server try { - Console::success("Creating Runtime Server"); - $this->triggerCreateRuntimeServer($functionId, $projectId, $tagId, $dbForProject); + Console::info("[ INFO ] Creating runtime server"); + $this->triggerCreateRuntimeServer($projectId, $functionId, $tagId, $dbForProject); } catch (\Throwable $th) { Console::error($th->getMessage()); $tag->setAttribute('status', 'failed'); @@ -252,7 +248,7 @@ class BuildsV1 extends Worker return; } - Console::success("Runtime Server created successfully"); + Console::success("Runtime Server created"); } public function shutdown(): void