CodeMirror/appwrite
1
0
Fork 0
mirror of synced 2024-06-29 19:50:26 +12:00
Code Issues Projects Releases Wiki Activity

feat: handle build and execution errors

Browse source
This commit is contained in:
Christy Jacob 2022-02-15 21:39:03 +04:00
parent 1d6d43baf8
commit 123c47fb9a
4 changed files with 142 additions and 174 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

50
app/controllers/api/functions.php
View file

@ -864,32 +864,40 @@ App::post('/v1/functions/:functionId/executions')
/** Execute function */ /** Execute function */
$executor = new Executor(); $executor = new Executor();
$responseExecute = $executor->createExecution( $executionResponse = [];
projectId: $project->getId(), try {
functionId: $function->getId(), $executionResponse = $executor->createExecution(
deploymentId: $deployment->getId(), projectId: $project->getId(),
path: $build->getAttribute('outputPath', ''), functionId: $function->getId(),
vars: $vars, deploymentId: $deployment->getId(),
data: $data, path: $build->getAttribute('outputPath', ''),
entrypoint: $deployment->getAttribute('entrypoint', ''), vars: $vars,
runtime: $function->getAttribute('runtime', ''), data: $data,
timeout: $function->getAttribute('timeout', 0), entrypoint: $deployment->getAttribute('entrypoint', ''),
baseImage: $runtime['image'] runtime: $function->getAttribute('runtime', ''),
); timeout: $function->getAttribute('timeout', 0),
baseImage: $runtime['image']
);
/** Update execution status */
$execution->setAttribute('status', $executionResponse['status']);
$execution->setAttribute('statusCode', $executionResponse['statusCode']);
$execution->setAttribute('stdout', $executionResponse['stdout']);
$execution->setAttribute('stderr', $executionResponse['stderr']);
$execution->setAttribute('time', $executionResponse['time']);
} catch (\Throwable $th) {
$execution->setAttribute('status', 'failed');
$execution->setAttribute('statusCode', $th->getCode());
$execution->setAttribute('stderr', $th->getMessage());
Console::error($th->getMessage());
}
/** Update execution status */
$execution->setAttribute('status', $responseExecute['status']);
$execution->setAttribute('statusCode', $responseExecute['statusCode']);
$execution->setAttribute('stdout', $responseExecute['stdout']);
$execution->setAttribute('stderr', $responseExecute['stderr']);
$execution->setAttribute('time', $responseExecute['time']);
Authorization::skip(fn() => $dbForProject->updateDocument('executions', $executionId, $execution)); Authorization::skip(fn() => $dbForProject->updateDocument('executions', $executionId, $execution));
$executionResponse['response'] = ($executionResponse['status'] !== 'completed') ? $executionResponse['stderr'] : $executionResponse['stdout'];
$responseExecute['response'] = ($responseExecute['status'] !== 'completed') ? $responseExecute['stderr'] : $responseExecute['stdout'];
$response $response
->setStatusCode(Response::STATUS_CODE_CREATED) ->setStatusCode(Response::STATUS_CODE_CREATED)
->dynamic(new Document($responseExecute), Response::MODEL_SYNC_EXECUTION); ->dynamic(new Document($executionResponse), Response::MODEL_SYNC_EXECUTION);
}); });
App::get('/v1/functions/:functionId/executions') App::get('/v1/functions/:functionId/executions')

202
app/executor.php
View file

@ -35,9 +35,10 @@ use Utopia\Validator\Text;
// Remove builds param from delete endpoint - done // Remove builds param from delete endpoint - done
// Shutdown callback isn't working as expected - done // Shutdown callback isn't working as expected - done
// Incorporate Matej's changes in the build stage ( moving of the tar file will be performed by the runtime and not the build stage ) // Fix error handling
// Fix error handling and logging // Fix logging
// Fix delete endpoint // Fix delete endpoint
// Incorporate Matej's changes in the build stage ( moving of the tar file will be performed by the runtime and not the build stage )
// Add size validators for the runtime IDs // Add size validators for the runtime IDs
// Decide on logic for build and runtime containers names ( runtime-ID and build-ID) // Decide on logic for build and runtime containers names ( runtime-ID and build-ID)
@ -126,10 +127,14 @@ App::post('/v1/runtimes')
->action(function (string $runtimeId, string $source, string $destination, array $vars, string $runtime, string $baseImage, $orchestrationPool, $activeRuntimes, Response $response) { ->action(function (string $runtimeId, string $source, string $destination, array $vars, string $runtime, string $baseImage, $orchestrationPool, $activeRuntimes, Response $response) {
// TODO: Check if runtime already exists.. // TODO: Check if runtime already exists..
$orchestration = $orchestrationPool->get(); $container = 'runtime-' . $runtimeId;
if ($activeRuntimes->exists($container)) {
throw new Exception('Runtime already exists.', 409);
}
$build = []; $build = [];
$id = ''; $buildId = '';
$buildStdout = ''; $buildStdout = '';
$buildStderr = ''; $buildStderr = '';
$buildStart = \time(); $buildStart = \time();
@ -150,7 +155,7 @@ App::post('/v1/runtimes')
$device = new Local($destination); $device = new Local($destination);
$buffer = $device->read($source); $buffer = $device->read($source);
if(!$device->write($tmpSource, $buffer)) { if(!$device->write($tmpSource, $buffer)) {
throw new Exception('Failed to write source code to temporary location.', 500); throw new Exception('Failed to copy source code to temporary directory', 500);
}; };
/** /**
@ -165,6 +170,7 @@ App::post('/v1/runtimes')
/** /**
* Create container * Create container
*/ */
$orchestration = $orchestrationPool->get();
$container = 'build-' . $runtimeId; $container = 'build-' . $runtimeId;
$vars = array_map(fn ($v) => strval($v), $vars); $vars = array_map(fn ($v) => strval($v), $vars);
$orchestration $orchestration
@ -172,7 +178,7 @@ App::post('/v1/runtimes')
->setMemory(App::getEnv('_APP_FUNCTIONS_MEMORY', 256)) ->setMemory(App::getEnv('_APP_FUNCTIONS_MEMORY', 256))
->setSwap(App::getEnv('_APP_FUNCTIONS_MEMORY_SWAP', 256)); ->setSwap(App::getEnv('_APP_FUNCTIONS_MEMORY_SWAP', 256));
$id = $orchestration->run( $buildId = $orchestration->run(
image: $baseImage, image: $baseImage,
name: $container, name: $container,
vars: $vars, vars: $vars,
@ -195,7 +201,7 @@ App::post('/v1/runtimes')
] ]
); );
if (empty($id)) { if (empty($buildId)) {
throw new Exception('Failed to create build container', 500); throw new Exception('Failed to create build container', 500);
} }
@ -217,7 +223,7 @@ App::post('/v1/runtimes')
); );
if (!$untarSuccess) { if (!$untarSuccess) {
throw new Exception('Failed to extract tar: ' . $untarStderr); throw new Exception('Failed to extract tarfile ' . $untarStderr, 500);
} }
/** /**
@ -232,7 +238,7 @@ App::post('/v1/runtimes')
); );
if (!$buildSuccess) { if (!$buildSuccess) {
throw new Exception('Failed to build dependencies: ' . $buildStderr); throw new Exception('Failed to build dependencies: ' . $buildStderr, 500);
} }
/** /**
@ -251,12 +257,12 @@ App::post('/v1/runtimes')
); );
if (!$compressSuccess) { if (!$compressSuccess) {
throw new Exception('Failed to compress built code: ' . $compressStderr); throw new Exception('Failed to compress built code: ' . $compressStderr, 500);
} }
// Check if the build was successful by checking if file exists // Check if the build was successful by checking if file exists
if (!\file_exists($tmpBuild)) { if (!\file_exists($tmpBuild)) {
throw new Exception('Something went wrong during the build process.'); throw new Exception('Something went wrong during the build process');
} }
/** /**
@ -288,116 +294,90 @@ App::post('/v1/runtimes')
'endTime' => $buildEnd, 'endTime' => $buildEnd,
'duration' => $buildEnd - $buildStart, 'duration' => $buildEnd - $buildStart,
]; ];
Console::success('Build Stage Ran in ' . ($buildEnd - $buildStart) . ' seconds');
Console::success('Build Stage completed in ' . ($buildEnd - $buildStart) . ' seconds');
} catch (Throwable $th) { } catch (Throwable $th) {
$buildEnd = \time();
$buildStderr = $th->getMessage();
$build = [
'status' => 'failed',
// Increase logs limit
'stdout' => \utf8_encode(\mb_substr($buildStdout, -4096)),
'stderr' => \utf8_encode(\mb_substr($buildStderr, -4096)),
'startTime' => $buildStart,
'endTime' => $buildEnd,
'duration' => $buildEnd - $buildStart,
];
Console::error('Build failed: ' . $th->getMessage()); Console::error('Build failed: ' . $th->getMessage());
throw new Exception($th->getMessage(), 500);
} finally { } finally {
if (!empty($id)) { if (!empty($buildId)) {
$orchestration->remove($id, true); $orchestration->remove($buildId, true);
} }
} $orchestrationPool->put($orchestration);
if ( $build['status'] !== 'ready') {
return $response
->setStatusCode(500)
->json($build);
} }
/** Create runtime server */ /** Create runtime server */
try { try {
$container = 'runtime-' . $runtimeId; $orchestration = $orchestrationPool->get();
if ($activeRuntimes->exists($container) && !(\substr($activeRuntimes->get($container)['status'], 0, 2) === 'Up')) { // Remove container if not online
// If container is online then stop and remove it
try {
$orchestration->remove($container, true);
} catch (Exception $e) {
throw new Exception('Failed to remove container: ' . $e->getMessage());
}
$activeRuntimes->del($container);
}
/** /**
* Copy code files from source to a temporary location on the executor * Copy code files from source to a temporary location on the executor
*/ */
$buffer = $device->read($outputPath); $buffer = $device->read($outputPath);
if(!$device->write($tmpBuild, $buffer)) { if(!$device->write($tmpBuild, $buffer)) {
throw new Exception('Failed to write built code to temporary location.', 500); throw new Exception('Failed to copy built code to temporary location.', 500);
}; };
/** /**
* Launch Runtime * Launch Runtime
*/ */
$container = 'runtime-' . $runtimeId;
$secret = \bin2hex(\random_bytes(16)); $secret = \bin2hex(\random_bytes(16));
$vars = \array_merge($vars, [ $vars = \array_merge($vars, [
'INTERNAL_RUNTIME_KEY' => $secret 'INTERNAL_RUNTIME_KEY' => $secret
]); ]);
if (!$activeRuntimes->exists($container)) { $executionStart = \microtime(true);
$executionStart = \microtime(true); $executionTime = \time();
$executionTime = \time();
$vars = array_map(fn ($v) => strval($v), $vars);
$vars = array_map(fn ($v) => strval($v), $vars);
$orchestration
$orchestration ->setCpus(App::getEnv('_APP_FUNCTIONS_CPUS', '1'))
->setCpus(App::getEnv('_APP_FUNCTIONS_CPUS', '1')) ->setMemory(App::getEnv('_APP_FUNCTIONS_MEMORY', '256'))
->setMemory(App::getEnv('_APP_FUNCTIONS_MEMORY', '256')) ->setSwap(App::getEnv('_APP_FUNCTIONS_MEMORY_SWAP', '256'));
->setSwap(App::getEnv('_APP_FUNCTIONS_MEMORY_SWAP', '256'));
$id = $orchestration->run(
$id = $orchestration->run( image: $baseImage,
image: $baseImage, name: $container,
name: $container, vars: $vars,
vars: $vars, labels: [
labels: [ 'openruntimes-id' => $runtimeId,
'openruntimes-id' => $runtimeId, 'openruntimes-type' => 'function',
'openruntimes-type' => 'function', 'openruntimes-created' => strval($executionTime),
'openruntimes-created' => strval($executionTime), 'openruntimes-runtime' => $runtime
'openruntimes-runtime' => $runtime ],
], hostname: $container,
hostname: $container, mountFolder: \dirname($tmpBuild),
mountFolder: \dirname($tmpBuild), );
);
if (empty($id)) {
if (empty($id)) { throw new Exception('Failed to create runtime', 500);
throw new Exception('Failed to create container');
}
// Add to network
$orchestration->networkConnect($container, App::getEnv('_APP_EXECUTOR_RUNTIME_NETWORK', 'appwrite_runtimes'));
$executionEnd = \microtime(true);
$activeRuntimes->set($container, [
'id' => $id,
'name' => $container,
'created' => $executionTime,
'updated' => $executionTime,
'status' => 'Up ' . \round($executionEnd - $executionStart, 2) . 's',
'key' => $secret,
]);
} }
$orchestration->networkConnect($container, App::getEnv('_APP_EXECUTOR_RUNTIME_NETWORK', 'openruntimes'));
$executionEnd = \microtime(true);
$activeRuntimes->set($container, [
'id' => $id,
'name' => $container,
'created' => $executionTime,
'updated' => $executionTime,
'status' => 'Up ' . \round($executionEnd - $executionStart, 2) . 's',
'key' => $secret,
]);
Console::success('Runtime Server created in ' . ($executionEnd - $executionStart) . ' seconds'); Console::success('Runtime Server created in ' . ($executionEnd - $executionStart) . ' seconds');
} catch (\Throwable $th) { } catch (\Throwable $th) {
Console::error('Runtime Server Creation Failed: '. $th->getMessage()); Console::error('Runtime Server Creation Failed: '. $th->getMessage());
throw new Exception($th->getMessage(), 500);
} finally {
$orchestrationPool->put($orchestration);
} }
$orchestrationPool->put($orchestration);
$response $response
->setStatusCode(201) ->setStatusCode(Response::STATUS_CODE_CREATED)
->json($build); ->json($build);
}); });
@ -413,7 +393,6 @@ App::get('/v1/runtimes')
$runtimes[] = $runtime; $runtimes[] = $runtime;
} }
// TODO: Response model for runtimes and runtimes list
$response $response
->setStatusCode(200) ->setStatusCode(200)
->json($runtimes); ->json($runtimes);
@ -421,7 +400,6 @@ App::get('/v1/runtimes')
App::get('/v1/runtimes/:runtimeId') App::get('/v1/runtimes/:runtimeId')
->desc("Get a runtime by its ID") ->desc("Get a runtime by its ID")
// Change the text validators to UID
->param('runtimeId', '', new Text(128), 'Runtime unique ID.') ->param('runtimeId', '', new Text(128), 'Runtime unique ID.')
->inject('activeRuntimes') ->inject('activeRuntimes')
->inject('response') ->inject('response')
@ -447,18 +425,19 @@ App::delete('/v1/runtimes/:runtimeId')
->action(function (string $runtimeId, $orchestrationPool, $activeRuntimes, Response $response) { ->action(function (string $runtimeId, $orchestrationPool, $activeRuntimes, Response $response) {
$container = 'runtime-' . $runtimeId; $container = 'runtime-' . $runtimeId;
if(!$activeRuntimes->exists($container)) {
throw new Exception('Runtime not found', 404);
}
Console::info('Deleting runtime: ' . $container); Console::info('Deleting runtime: ' . $container);
try { try {
$orchestration = $orchestrationPool->get(); $orchestration = $orchestrationPool->get();
$status = $orchestration->remove($container, true); $orchestration->remove($container, true);
if ($status) {
Console::success('Removed runtime container: ' . $runtimeId);
} else {
Console::error('Failed to remove runtime container: ' . $runtimeId);
}
$activeRuntimes->del($container); $activeRuntimes->del($container);
Console::success('Removed runtime container: ' . $container);
} catch (\Throwable $th) { } catch (\Throwable $th) {
Console::error('Failed to remove runtime container: ' . $runtimeId);
} finally { } finally {
$orchestrationPool->put($orchestration); $orchestrationPool->put($orchestration);
} }
@ -621,40 +600,27 @@ App::setResource('orchestrationPool', fn() => $orchestrationPool);
App::setResource('activeRuntimes', fn() => $activeRuntimes); App::setResource('activeRuntimes', fn() => $activeRuntimes);
/** Set callbacks */ /** Set callbacks */
App::error(function ($error, $utopia, $request, $response) { App::error(function ($error, $response) {
/** @var Exception $error */ // $route = $utopia->match($request);
/** @var Utopia\App $utopia */
/** @var Utopia\Swoole\Request $request */
$route = $utopia->match($request);
// logError($error, "httpError", $route); // logError($error, "httpError", $route);
$version = App::getEnv('_APP_VERSION', 'UNKNOWN'); $output = [
$code = $error->getCode();
$message = $error->getMessage();
$output = ((App::isDevelopment())) ? [
'message' => $error->getMessage(), 'message' => $error->getMessage(),
'code' => $error->getCode(), 'code' => $error->getCode(),
'file' => $error->getFile(), 'file' => $error->getFile(),
'line' => $error->getLine(), 'line' => $error->getLine(),
'trace' => $error->getTrace(), 'trace' => $error->getTrace(),
'version' => $version, 'version' => App::getEnv('_APP_VERSION', 'UNKNOWN'),
] : [
'message' => $message,
'code' => $code,
'version' => $version,
]; ];
$response $response
->addHeader('Cache-Control', 'no-cache, no-store, must-revalidate') ->addHeader('Cache-Control', 'no-cache, no-store, must-revalidate')
->addHeader('Expires', '0') ->addHeader('Expires', '0')
->addHeader('Pragma', 'no-cache') ->addHeader('Pragma', 'no-cache')
->setStatusCode(500); ->setStatusCode($error->getCode());
$response->json($output); $response->json($output);
}, ['error', 'utopia', 'request', 'response']); }, ['error', 'response']);
App::init(function ($request, $response) { App::init(function ($request, $response) {
$secretKey = $request->getHeader('x-appwrite-executor-key', ''); $secretKey = $request->getHeader('x-appwrite-executor-key', '');

60
app/workers/functions.php
View file

@ -14,28 +14,12 @@ use Utopia\Config\Config;
use Utopia\Database\Database; use Utopia\Database\Database;
use Utopia\Database\Document; use Utopia\Database\Document;
use Utopia\Database\Validator\Authorization; use Utopia\Database\Validator\Authorization;
use Utopia\Orchestration\Adapter\DockerAPI;
use Utopia\Orchestration\Orchestration;
require_once __DIR__.'/../init.php'; require_once __DIR__.'/../init.php';
Runtime::enableCoroutine(0);
Console::title('Functions V1 Worker'); Console::title('Functions V1 Worker');
Console::success(APP_NAME . ' functions worker v1 has started'); Console::success(APP_NAME . ' functions worker v1 has started');
$runtimes = Config::getParam('runtimes');
$dockerUser = App::getEnv('DOCKERHUB_PULL_USERNAME', null);
$dockerPass = App::getEnv('DOCKERHUB_PULL_PASSWORD', null);
$dockerEmail = App::getEnv('DOCKERHUB_PULL_EMAIL', null);
$orchestration = new Orchestration(new DockerAPI($dockerUser, $dockerPass, $dockerEmail));
$warmupEnd = \microtime(true);
$warmupTime = $warmupEnd - $warmupStart;
Console::success('Finished warmup in ' . $warmupTime . ' seconds');
class FunctionsV1 extends Worker class FunctionsV1 extends Worker
{ {
/** /**
@ -303,25 +287,33 @@ class FunctionsV1 extends Worker
$vars = \array_merge($function->getAttribute('vars', []), $vars); $vars = \array_merge($function->getAttribute('vars', []), $vars);
/** Execute function */ /** Execute function */
$executionResponse = $this->executor->createExecution( try {
projectId: $projectId, $executionResponse = $this->executor->createExecution(
functionId: $functionId, projectId: $projectId,
deploymentId: $deploymentId, functionId: $functionId,
path: $build->getAttribute('outputPath', ''), deploymentId: $deploymentId,
vars: $vars, path: $build->getAttribute('outputPath', ''),
entrypoint: $deployment->getAttribute('entrypoint', ''), vars: $vars,
data: $vars['APPWRITE_FUNCTION_DATA'], entrypoint: $deployment->getAttribute('entrypoint', ''),
runtime: $function->getAttribute('runtime', ''), data: $vars['APPWRITE_FUNCTION_DATA'],
timeout: $function->getAttribute('timeout', 0), runtime: $function->getAttribute('runtime', ''),
baseImage: $runtime['image'] timeout: $function->getAttribute('timeout', 0),
); baseImage: $runtime['image']
);
/** Update execution status */
$execution->setAttribute('status', $executionResponse['status']);
$execution->setAttribute('statusCode', $executionResponse['statusCode']);
$execution->setAttribute('stdout', $executionResponse['stdout']);
$execution->setAttribute('stderr', $executionResponse['stderr']);
$execution->setAttribute('time', $executionResponse['time']);
} catch (\Throwable $th) {
$execution->setAttribute('status', 'failed');
$execution->setAttribute('statusCode', $th->getCode());
$execution->setAttribute('stderr', $th->getMessage());
Console::error($th->getMessage());
}
/** Update execution status */
$execution->setAttribute('status', $executionResponse['status']);
$execution->setAttribute('statusCode', $executionResponse['statusCode']);
$execution->setAttribute('stdout', $executionResponse['stdout']);
$execution->setAttribute('stderr', $executionResponse['stderr']);
$execution->setAttribute('time', $executionResponse['time']);
$execution = Authorization::skip(fn() => $dbForProject->updateDocument('executions', $executionId, $execution)); $execution = Authorization::skip(fn() => $dbForProject->updateDocument('executions', $executionId, $execution));
/** Trigger Webhook */ /** Trigger Webhook */

4
src/Executor/Executor.php
View file

@ -54,6 +54,7 @@ class Executor
'baseImage' => $baseImage 'baseImage' => $baseImage
]; ];
var_dump($params);
$response = $this->call(self::METHOD_POST, $route, $headers, $params, true, 30); $response = $this->call(self::METHOD_POST, $route, $headers, $params, true, 30);
$status = $response['headers']['status-code']; $status = $response['headers']['status-code'];
@ -115,7 +116,8 @@ class Executor
'timeout' => $timeout, 'timeout' => $timeout,
'baseImage' => $baseImage, 'baseImage' => $baseImage,
]; ];
var_dump($params);
$response = $this->call(self::METHOD_POST, $route, $headers, $params, true, 30); $response = $this->call(self::METHOD_POST, $route, $headers, $params, true, 30);
$status = $response['headers']['status-code']; $status = $response['headers']['status-code'];