feat: refactoring the builds worker
This commit is contained in:
parent
046797e48f
commit
074a293d63
3 changed files with 29 additions and 35 deletions
|
@ -151,22 +151,15 @@ function createRuntimeServer(string $functionId, string $projectId, string $tagI
|
||||||
global $runtimes;
|
global $runtimes;
|
||||||
global $activeFunctions;
|
global $activeFunctions;
|
||||||
|
|
||||||
var_dump("Im here 1");
|
|
||||||
|
|
||||||
try {
|
try {
|
||||||
$orchestration = $orchestrationPool->get();
|
$orchestration = $orchestrationPool->get();
|
||||||
var_dump("Im here 2");
|
|
||||||
$function = $database->getDocument('functions', $functionId);
|
$function = $database->getDocument('functions', $functionId);
|
||||||
var_dump("Im here 3");
|
|
||||||
$tag = $database->getDocument('tags', $tagId);
|
$tag = $database->getDocument('tags', $tagId);
|
||||||
var_dump("Im here 4");
|
|
||||||
|
|
||||||
if ($tag->getAttribute('buildId') === null) {
|
if ($tag->getAttribute('buildId') === null) {
|
||||||
throw new Exception('Tag has no buildId');
|
throw new Exception('Tag has no buildId');
|
||||||
}
|
}
|
||||||
|
|
||||||
var_dump("Im here 5");
|
|
||||||
|
|
||||||
// Grab Build Document
|
// Grab Build Document
|
||||||
$build = $database->getDocument('builds', $tag->getAttribute('buildId'));
|
$build = $database->getDocument('builds', $tag->getAttribute('buildId'));
|
||||||
|
|
||||||
|
@ -177,8 +170,6 @@ function createRuntimeServer(string $functionId, string $projectId, string $tagI
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
var_dump("Im here");
|
|
||||||
|
|
||||||
// Generate random secret key
|
// Generate random secret key
|
||||||
$secret = \bin2hex(\random_bytes(16));
|
$secret = \bin2hex(\random_bytes(16));
|
||||||
|
|
||||||
|
@ -318,7 +309,8 @@ function createRuntimeServer(string $functionId, string $projectId, string $tagI
|
||||||
Console::info('Runtime server is ready to run');
|
Console::info('Runtime server is ready to run');
|
||||||
}
|
}
|
||||||
} catch (\Throwable $th) {
|
} catch (\Throwable $th) {
|
||||||
$orchestrationPool->put($orchestration);
|
var_dump($th->getTraceAsString());
|
||||||
|
$orchestrationPool->put($orchestration ?? null);
|
||||||
throw $th;
|
throw $th;
|
||||||
} finally {
|
} finally {
|
||||||
$orchestrationPool->put($orchestration);
|
$orchestrationPool->put($orchestration);
|
||||||
|
@ -797,7 +789,7 @@ App::post('/v1/cleanup/tag')
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
||||||
App::post('/v1/create/runtime')
|
App::post('/v1/executor/runtime')
|
||||||
->desc('Create a new runtime server')
|
->desc('Create a new runtime server')
|
||||||
->param('functionId', '', new UID(), 'Function unique ID.')
|
->param('functionId', '', new UID(), 'Function unique ID.')
|
||||||
->param('tagId', '', new UID(), 'Tag unique ID.')
|
->param('tagId', '', new UID(), 'Tag unique ID.')
|
||||||
|
@ -857,6 +849,7 @@ App::post('/v1/build/:buildId') // Start a Build
|
||||||
throw new Exception('Build is already finished', 409);
|
throw new Exception('Build is already finished', 409);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Console::success('Starting build ' . $buildId);
|
||||||
// Build Code
|
// Build Code
|
||||||
runBuildStage($buildId, $projectID, $dbForProject);
|
runBuildStage($buildId, $projectID, $dbForProject);
|
||||||
|
|
||||||
|
|
|
@ -21,6 +21,11 @@ use Utopia\Logger\Log\User;
|
||||||
|
|
||||||
$http = new Server("0.0.0.0", App::getEnv('PORT', 80));
|
$http = new Server("0.0.0.0", App::getEnv('PORT', 80));
|
||||||
|
|
||||||
|
$http->set([
|
||||||
|
'worker_num' => 1,
|
||||||
|
'reactor_num' => 6
|
||||||
|
]);
|
||||||
|
|
||||||
$payloadSize = max(4000000 /* 4mb */, App::getEnv('_APP_STORAGE_LIMIT', 10000000 /* 10mb */));
|
$payloadSize = max(4000000 /* 4mb */, App::getEnv('_APP_STORAGE_LIMIT', 10000000 /* 10mb */));
|
||||||
|
|
||||||
$http
|
$http
|
||||||
|
|
|
@ -11,9 +11,13 @@ use Utopia\Config\Config;
|
||||||
|
|
||||||
require_once __DIR__.'/../init.php';
|
require_once __DIR__.'/../init.php';
|
||||||
|
|
||||||
|
// Disable Auth since we already validate it in the API
|
||||||
|
Authorization::disable();
|
||||||
|
|
||||||
Console::title('Builds V1 Worker');
|
Console::title('Builds V1 Worker');
|
||||||
Console::success(APP_NAME.' build worker v1 has started');
|
Console::success(APP_NAME.' build worker v1 has started');
|
||||||
|
|
||||||
|
// TODO: Executor should return appropriate response codes.
|
||||||
class BuildsV1 extends Worker
|
class BuildsV1 extends Worker
|
||||||
{
|
{
|
||||||
|
|
||||||
|
@ -35,14 +39,14 @@ class BuildsV1 extends Worker
|
||||||
case BUILD_TYPE_TAG:
|
case BUILD_TYPE_TAG:
|
||||||
$functionId = $this->args['functionId'] ?? '';
|
$functionId = $this->args['functionId'] ?? '';
|
||||||
$tagId = $this->args['tagId'] ?? '';
|
$tagId = $this->args['tagId'] ?? '';
|
||||||
Console::success("Creating build for tag: $tagId");
|
Console::info("[ INFO ] Creating build for tag: $tagId");
|
||||||
$this->buildTag($projectId, $functionId, $tagId);
|
$this->buildTag($projectId, $functionId, $tagId);
|
||||||
break;
|
break;
|
||||||
|
|
||||||
case BUILD_TYPE_RETRY:
|
case BUILD_TYPE_RETRY:
|
||||||
$buildId = $this->args['buildId'] ?? '';
|
$buildId = $this->args['buildId'] ?? '';
|
||||||
Console::success("Retrying build for id: $buildId");
|
Console::info("[ INFO ] Retrying build for id: $buildId");
|
||||||
$this->triggerBuildStage($projectId, $buildId,);
|
$this->triggerBuild($projectId, $buildId);
|
||||||
break;
|
break;
|
||||||
|
|
||||||
default:
|
default:
|
||||||
|
@ -51,7 +55,7 @@ class BuildsV1 extends Worker
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
protected function triggerBuildStage(string $projectId, string $buildId)
|
protected function triggerBuild(string $projectId, string $buildId)
|
||||||
{
|
{
|
||||||
// TODO: What is a reasonable time to wait for a build to complete?
|
// TODO: What is a reasonable time to wait for a build to complete?
|
||||||
$ch = \curl_init();
|
$ch = \curl_init();
|
||||||
|
@ -95,7 +99,7 @@ class BuildsV1 extends Worker
|
||||||
protected function triggerCreateRuntimeServer(string $projectId, string $functionId, string $tagId)
|
protected function triggerCreateRuntimeServer(string $projectId, string $functionId, string $tagId)
|
||||||
{
|
{
|
||||||
$ch = \curl_init();
|
$ch = \curl_init();
|
||||||
\curl_setopt($ch, CURLOPT_URL, "http://appwrite-executor:8080/v1/create/runtime");
|
\curl_setopt($ch, CURLOPT_URL, "http://appwrite-executor:8080/v1/executor/runtime");
|
||||||
\curl_setopt($ch, CURLOPT_POST, true);
|
\curl_setopt($ch, CURLOPT_POST, true);
|
||||||
\curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
|
\curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
|
||||||
\curl_setopt($ch, CURLOPT_TIMEOUT, 900);
|
\curl_setopt($ch, CURLOPT_TIMEOUT, 900);
|
||||||
|
@ -120,7 +124,7 @@ class BuildsV1 extends Worker
|
||||||
|
|
||||||
\curl_close($ch);
|
\curl_close($ch);
|
||||||
|
|
||||||
if ($responseStatus !== 200) {
|
if ($responseStatus >= 400) {
|
||||||
throw new \Exception("Build failed with status code: $responseStatus");
|
throw new \Exception("Build failed with status code: $responseStatus");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -128,20 +132,13 @@ class BuildsV1 extends Worker
|
||||||
if (isset($response['error'])) {
|
if (isset($response['error'])) {
|
||||||
throw new \Exception($response['error']);
|
throw new \Exception($response['error']);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (isset($response['success']) && $response['success'] === true) {
|
|
||||||
return;
|
|
||||||
} else {
|
|
||||||
throw new \Exception("Build failed");
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
protected function buildTag(string $projectId, string $functionId, string $tagId)
|
protected function buildTag(string $projectId, string $functionId, string $tagId)
|
||||||
{
|
{
|
||||||
$dbForProject = $this->getProjectDB($projectId);
|
$dbForProject = $this->getProjectDB($projectId);
|
||||||
|
|
||||||
// TODO: Why does it need to skip authorization?
|
$function = $dbForProject->getDocument('functions', $functionId);
|
||||||
$function = Authorization::skip(fn() => $dbForProject->getDocument('functions', $functionId));
|
|
||||||
|
|
||||||
if ($function->isEmpty()) {
|
if ($function->isEmpty()) {
|
||||||
throw new Exception('Function not found', 404);
|
throw new Exception('Function not found', 404);
|
||||||
|
@ -170,8 +167,8 @@ class BuildsV1 extends Worker
|
||||||
// TODO : What should be the read and write permissions for a build ?
|
// TODO : What should be the read and write permissions for a build ?
|
||||||
$dbForProject->createDocument('builds', new Document([
|
$dbForProject->createDocument('builds', new Document([
|
||||||
'$id' => $buildId,
|
'$id' => $buildId,
|
||||||
'$read' => ['role:all'],
|
'$read' => [],
|
||||||
'$write' => ['role:all'],
|
'$write' => [],
|
||||||
'dateCreated' => time(),
|
'dateCreated' => time(),
|
||||||
'status' => 'processing',
|
'status' => 'processing',
|
||||||
'runtime' => $function->getAttribute('runtime'),
|
'runtime' => $function->getAttribute('runtime'),
|
||||||
|
@ -205,10 +202,10 @@ class BuildsV1 extends Worker
|
||||||
|
|
||||||
// Build the Code
|
// Build the Code
|
||||||
try {
|
try {
|
||||||
Console::success("Creating Build with id: $buildId");
|
Console::info("[ INFO ] Creating build with id: $buildId");
|
||||||
$tag->setAttribute('status', 'building');
|
$tag->setAttribute('status', 'building');
|
||||||
$tag = $dbForProject->updateDocument('tags', $tagId, $tag);
|
$tag = $dbForProject->updateDocument('tags', $tagId, $tag);
|
||||||
$this->triggerBuildStage($projectId, $buildId);
|
$this->triggerBuild($projectId, $buildId);
|
||||||
} catch (\Throwable $th) {
|
} catch (\Throwable $th) {
|
||||||
Console::error($th->getMessage());
|
Console::error($th->getMessage());
|
||||||
$tag->setAttribute('status', 'failed');
|
$tag->setAttribute('status', 'failed');
|
||||||
|
@ -216,7 +213,7 @@ class BuildsV1 extends Worker
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
Console::success("Build id: $buildId completed successfully");
|
Console::success("[ SUCCESS ] Build id: $buildId completed");
|
||||||
|
|
||||||
// Update the schedule
|
// Update the schedule
|
||||||
$schedule = $function->getAttribute('schedule', '');
|
$schedule = $function->getAttribute('schedule', '');
|
||||||
|
@ -237,14 +234,13 @@ class BuildsV1 extends Worker
|
||||||
->setAttribute('tag', $tag->getId())
|
->setAttribute('tag', $tag->getId())
|
||||||
->setAttribute('scheduleNext', (int)$next);
|
->setAttribute('scheduleNext', (int)$next);
|
||||||
|
|
||||||
// TODO: Why should we disable auth ?
|
$function = $dbForProject->updateDocument('functions', $functionId, $function);
|
||||||
$function = Authorization::skip(fn() => $dbForProject->updateDocument('functions', $functionId, $function));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Deploy Runtime Server
|
// Deploy Runtime Server
|
||||||
try {
|
try {
|
||||||
Console::success("Creating Runtime Server");
|
Console::info("[ INFO ] Creating runtime server");
|
||||||
$this->triggerCreateRuntimeServer($functionId, $projectId, $tagId, $dbForProject);
|
$this->triggerCreateRuntimeServer($projectId, $functionId, $tagId, $dbForProject);
|
||||||
} catch (\Throwable $th) {
|
} catch (\Throwable $th) {
|
||||||
Console::error($th->getMessage());
|
Console::error($th->getMessage());
|
||||||
$tag->setAttribute('status', 'failed');
|
$tag->setAttribute('status', 'failed');
|
||||||
|
@ -252,7 +248,7 @@ class BuildsV1 extends Worker
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
Console::success("Runtime Server created successfully");
|
Console::success("Runtime Server created");
|
||||||
}
|
}
|
||||||
|
|
||||||
public function shutdown(): void
|
public function shutdown(): void
|
||||||
|
|
Loading…
Reference in a new issue