diff --git a/.gitignore b/.gitignore
index 5433e2ba3a..3d6001ca06 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,6 +2,7 @@
/vendor/
/node_modules/
/tests/resources/storage/
+/tests/resources/functions/**/code.tar.gz
/app/sdks/*
/.idea/
.DS_Store
diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php
index bc85b2991e..6bd6167775 100644
--- a/app/controllers/api/functions.php
+++ b/app/controllers/api/functions.php
@@ -558,7 +558,9 @@ App::post('/v1/functions/:functionId/deployments')
throw new Exception('Failed moving file', 500);
}
- if ((bool) $activate) {
+ $activate = (bool) filter_var($activate, FILTER_VALIDATE_BOOLEAN);
+
+ if ($activate) {
// Remove deploy for all other deployments.
$deployments = $dbForProject->find('deployments', [
new Query('activate', Query::TYPE_EQUAL, [true]),
diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php
index d709c29c05..a9aef50a9b 100644
--- a/app/controllers/api/projects.php
+++ b/app/controllers/api/projects.php
@@ -601,7 +601,7 @@ App::post('/v1/projects/:projectId/webhooks')
throw new Exception('Project not found', 404, Exception::PROJECT_NOT_FOUND);
}
- $security = ($security === '1' || $security === 'true' || $security === 1 || $security === true);
+ $security = (bool) filter_var($security, FILTER_VALIDATE_BOOLEAN);
$webhook = new Document([
'$id' => $dbForConsole->getId(),
diff --git a/app/executor.php b/app/executor.php
index 2f5242e507..6a9c19585e 100644
--- a/app/executor.php
+++ b/app/executor.php
@@ -12,6 +12,7 @@ use Swoole\Timer;
use Utopia\App;
use Utopia\CLI\Console;
use Utopia\Logger\Log;
+use Utopia\Logger\Logger;
use Utopia\Orchestration\Adapter\DockerCLI;
use Utopia\Orchestration\Orchestration;
use Utopia\Storage\Device\Local;
@@ -20,29 +21,10 @@ use Utopia\Swoole\Request;
use Utopia\Swoole\Response;
use Utopia\Validator\ArrayList;
use Utopia\Validator\Assoc;
+use Utopia\Validator\Boolean;
use Utopia\Validator\Range as ValidatorRange;
use Utopia\Validator\Text;
-// TODO
-// Implement other endpoints - Done
-// Handle shutdown - Done
-// Get list of supported runtimes on startup - Done
-// Pull runtimes on startup -- Done
-// Move some logic to server start - Done
-// Add updated property to swoole table - Done
-// Clean up deployments older than X seconds - Done
-// Remove orphans on startup - done
-// Remove multiple request attempt to the runtime logic in executor - done
-// Remove builds param from delete endpoint - done
-// Shutdown callback isn't working as expected - done
-// Fix error handling - done
-// Decide on logic for build and runtime containers names ( runtime-ID and build-ID) - done
-// Add size validators for the runtime IDs - done
-
-
-// Fix logging
-// Fix delete endpoint
-// Incorporate Matej's changes in the build stage ( moving of the tar file will be performed by the runtime and not the build stage )
Runtime::enableCoroutine(true, SWOOLE_HOOK_ALL);
@@ -71,11 +53,23 @@ $orchestrationPool = new ConnectionPool(function () {
return $orchestration;
}, 10);
+
+/**
+ * Create logger instance
+ */
+$providerName = App::getEnv('_APP_LOGGING_PROVIDER', '');
+$providerConfig = App::getEnv('_APP_LOGGING_CONFIG', '');
+$logger = null;
+
+if(!empty($providerName) && !empty($providerConfig) && Logger::hasProvider($providerName)) {
+ $classname = '\\Utopia\\Logger\\Adapter\\'.\ucfirst($providerName);
+ $adapter = new $classname($providerConfig);
+ $logger = new Logger($adapter);
+}
+
function logError(Throwable $error, string $action, Utopia\Route $route = null)
{
- global $register;
-
- $logger = $register->get('logger');
+ global $logger;
if ($logger) {
$version = App::getEnv('_APP_VERSION', 'UNKNOWN');
@@ -116,55 +110,56 @@ function logError(Throwable $error, string $action, Utopia\Route $route = null)
App::post('/v1/runtimes')
->desc("Create a new runtime server")
- ->param('runtimeId', '', new Text(62), 'Unique runtime ID.')
+ ->param('runtimeId', '', new Text(64), 'Unique runtime ID.')
->param('source', '', new Text(0), 'Path to source files.')
- ->param('destination', '', new Text(0), 'Destination folder to store build files into.')
+ ->param('destination', '', new Text(0), 'Destination folder to store build files into.', true)
->param('vars', [], new Assoc(), 'Environment Variables required for the build')
->param('commands', [], new ArrayList(new Text(0)), 'Commands required to build the container')
->param('runtime', '', new Text(128), 'Runtime for the cloud function')
+ ->param('network', '', new Text(128), 'Network to attach the container to')
->param('baseImage', '', new Text(128), 'Base image name of the runtime')
+ ->param('entrypoint', '', new Text(256), 'Entrypoint of the code file', true)
+ ->param('remove', false, new Boolean(), 'Remove a runtime after execution')
+ ->param('workdir', '', new Text(256), 'Working directory', true)
->inject('orchestrationPool')
->inject('activeRuntimes')
->inject('response')
- ->action(function (string $runtimeId, string $source, string $destination, array $vars, array $commands, string $runtime, string $baseImage, $orchestrationPool, $activeRuntimes, Response $response) {
+ ->action(function (string $runtimeId, string $source, string $destination, array $vars, array $commands, string $runtime, string $network, string $baseImage, string $entrypoint, bool $remove, string $workdir, $orchestrationPool, $activeRuntimes, Response $response) {
- $container = 'r-' . $runtimeId;
-
- if ($activeRuntimes->exists($container)) {
+ if ($activeRuntimes->exists($runtimeId)) {
throw new Exception('Runtime already exists.', 409);
}
- $build = [];
- $buildId = '';
- $buildStdout = '';
- $buildStderr = '';
- $buildStart = \time();
- $buildEnd = 0;
+ $container = [];
+ $containerId = '';
+ $stdout = '';
+ $stderr = '';
+ $startTime = \time();
+ $endTime = 0;
try {
- Console::info('Building runtime with ID : ' . $runtimeId);
+ Console::info('Building container : ' . $runtimeId);
/**
* Temporary file paths in the executor
*/
$tmpSource = "/tmp/$runtimeId/code.tar.gz";
- $tmpBuildDir = "/tmp/$runtimeId/builds";
$tmpBuild = "/tmp/$runtimeId/builds/code.tar.gz";
/**
* Copy code files from source to a temporary location on the executor
*/
- $device = new Local($destination);
+ $device = new Local();
$buffer = $device->read($source);
if(!$device->write($tmpSource, $buffer)) {
throw new Exception('Failed to copy source code to temporary directory', 500);
};
/**
- * Create a temporary folder to store builds
+ * Create the mount folder
*/
- if (!\file_exists($tmpBuildDir)) {
- if (!@\mkdir($tmpBuildDir, 0755, true)) {
- throw new Exception("Can't create directory : $tmpBuildDir", 500);
+ if (!\file_exists(\dirname($tmpBuild))) {
+ if (!@\mkdir(\dirname($tmpBuild), 0755, true)) {
+ throw new Exception("Failed to create temporary directory", 500);
}
}
@@ -172,174 +167,128 @@ App::post('/v1/runtimes')
* Create container
*/
$orchestration = $orchestrationPool->get();
- $container = 'b-' . $runtimeId;
+ $secret = \bin2hex(\random_bytes(16));
+ $vars = \array_merge($vars, [
+ 'INTERNAL_RUNTIME_KEY' => $secret,
+ 'INTERNAL_RUNTIME_ENTRYPOINT' => $entrypoint,
+ ]);
$vars = array_map(fn ($v) => strval($v), $vars);
$orchestration
->setCpus(App::getEnv('_APP_FUNCTIONS_CPUS', 1))
->setMemory(App::getEnv('_APP_FUNCTIONS_MEMORY', 256))
->setSwap(App::getEnv('_APP_FUNCTIONS_MEMORY_SWAP', 256));
- $buildId = $orchestration->run(
+ /** Keep the container alive if we have commands to be executed */
+ $entrypoint = !empty($commands) ? [
+ 'tail',
+ '-f',
+ '/dev/null'
+ ] : [];
+
+ $containerId = $orchestration->run(
image: $baseImage,
- name: $container,
+ name: $runtimeId,
+ hostname: $runtimeId,
vars: $vars,
- workdir: '/usr/code',
+ command: $entrypoint,
labels: [
'openruntimes-id' => $runtimeId,
- 'openruntimes-type' => 'build',
- 'openruntimes-created' => strval($buildStart),
+ 'openruntimes-type' => 'runtime',
+ 'openruntimes-created' => strval($startTime),
'openruntimes-runtime' => $runtime,
],
- command: [
- 'tail',
- '-f',
- '/dev/null'
- ],
- hostname: $container,
- mountFolder: \dirname($tmpSource),
+ workdir: $workdir,
volumes: [
- "$tmpBuildDir:/usr/builds:rw"
+ \dirname($tmpSource). ':/tmp:rw',
+ \dirname($tmpBuild). ":/usr/code:rw"
]
);
- if (empty($buildId)) {
+ if (empty($containerId)) {
throw new Exception('Failed to create build container', 500);
}
- /**
- * Extract user code into build container
- */
- $status = $orchestration->execute(
- name: $container,
- command: $commands,
- stdout: $buildStdout,
- stderr: $buildStderr,
- timeout: App::getEnv('_APP_FUNCTIONS_TIMEOUT', 900)
- );
-
- if (!$status) {
- throw new Exception('Failed to build dependenices ' . $buildStderr, 500);
+ if (!empty($network)) {
+ $orchestration->networkConnect($runtimeId, $network);
}
- // Check if the build was successful by checking if file exists
- if (!\file_exists($tmpBuild)) {
- throw new Exception('Something went wrong during the build process', 500);
+ /**
+ * Execute any commands if they were provided
+ */
+ if (!empty($commands)) {
+ $status = $orchestration->execute(
+ name: $runtimeId,
+ command: $commands,
+ stdout: $stdout,
+ stderr: $stderr,
+ timeout: App::getEnv('_APP_FUNCTIONS_TIMEOUT', 900)
+ );
+
+ if (!$status) {
+ throw new Exception('Failed to build dependenices ' . $stderr, 500);
+ }
}
/**
* Move built code to expected build directory
*/
- $outputPath = $device->getPath(\uniqid() . '.' . \pathinfo('code.tar.gz', PATHINFO_EXTENSION));
+ if (!empty($destination)) {
+ // Check if the build was successful by checking if file exists
+ if (!\file_exists($tmpBuild)) {
+ throw new Exception('Something went wrong during the build process', 500);
+ }
- if (App::getEnv('_APP_STORAGE_DEVICE', Storage::DEVICE_LOCAL) === Storage::DEVICE_LOCAL) {
- if (!$device->move($tmpBuild, $outputPath)) {
+ $device = new Local($destination);
+ $outputPath = $device->getPath(\uniqid() . '.' . \pathinfo('code.tar.gz', PATHINFO_EXTENSION));
+
+ $buffer = $device->read($tmpBuild);
+ if(!$device->write($outputPath, $buffer)) {
throw new Exception('Failed to move built code to storage', 500);
- }
- } else {
- if (!$device->upload($tmpBuild, $outputPath)) {
- throw new Exception('Failed to upload built code upload to storage', 500);
- }
+ };
+
+ $container['outputPath'] = $outputPath;
}
- if ($buildStdout === '') {
- $buildStdout = 'Build Successful!';
+ if (empty($stdout)) {
+ $stdout = 'Build Successful!';
}
- $buildEnd = \time();
- $build = [
- 'outputPath' => $outputPath,
+ $endTime = \time();
+ $container = array_merge($container, [
'status' => 'ready',
- 'stdout' => \utf8_encode($buildStdout),
- 'stderr' => \utf8_encode($buildStderr),
- 'startTime' => $buildStart,
- 'endTime' => $buildEnd,
- 'duration' => $buildEnd - $buildStart,
- ];
+ 'stdout' => \utf8_encode($stdout),
+ 'stderr' => \utf8_encode($stderr),
+ 'startTime' => $startTime,
+ 'endTime' => $endTime,
+ 'duration' => $endTime - $startTime,
+ ]);
- Console::success('Build Stage completed in ' . ($buildEnd - $buildStart) . ' seconds');
+ if (!$remove) {
+ $activeRuntimes->set($runtimeId, [
+ 'id' => $containerId,
+ 'name' => $runtimeId,
+ 'created' => $startTime,
+ 'updated' => $endTime,
+ 'status' => 'Up ' . \round($endTime - $startTime, 2) . 's',
+ 'key' => $secret,
+ ]);
+ }
+
+ Console::success('Build Stage completed in ' . ($endTime - $startTime) . ' seconds');
} catch (Throwable $th) {
Console::error('Build failed: ' . $th->getMessage());
throw new Exception($th->getMessage(), 500);
} finally {
- if (!empty($buildId)) {
- $orchestration->remove($buildId, true);
+ if (!empty($containerId) && $remove) {
+ $orchestration->remove($containerId, true);
}
$orchestrationPool->put($orchestration);
}
- /** Create runtime server */
- try {
- $orchestration = $orchestrationPool->get();
- /**
- * Copy code files from source to a temporary location on the executor
- */
- $buffer = $device->read($outputPath);
- if(!$device->write($tmpBuild, $buffer)) {
- throw new Exception('Failed to copy built code to temporary location.', 500);
- };
-
- /**
- * Launch Runtime
- */
- $container = 'r-' . $runtimeId;
- $secret = \bin2hex(\random_bytes(16));
- $vars = \array_merge($vars, [
- 'INTERNAL_RUNTIME_KEY' => $secret
- ]);
-
- $executionStart = \microtime(true);
- $executionTime = \time();
-
- $vars = array_map(fn ($v) => strval($v), $vars);
-
- $orchestration
- ->setCpus(App::getEnv('_APP_FUNCTIONS_CPUS', 1))
- ->setMemory(App::getEnv('_APP_FUNCTIONS_MEMORY', 256))
- ->setSwap(App::getEnv('_APP_FUNCTIONS_MEMORY_SWAP', 256));
-
- $id = $orchestration->run(
- image: $baseImage,
- name: $container,
- vars: $vars,
- labels: [
- 'openruntimes-id' => $runtimeId,
- 'openruntimes-type' => 'function',
- 'openruntimes-created' => strval($executionTime),
- 'openruntimes-runtime' => $runtime
- ],
- hostname: $container,
- mountFolder: \dirname($tmpBuild),
- );
-
- if (empty($id)) {
- throw new Exception('Failed to create runtime', 500);
- }
-
- $orchestration->networkConnect($container, App::getEnv('_APP_EXECUTOR_RUNTIME_NETWORK', 'openruntimes'));
-
- $executionEnd = \microtime(true);
-
- $activeRuntimes->set($container, [
- 'id' => $id,
- 'name' => $container,
- 'created' => $executionTime,
- 'updated' => $executionTime,
- 'status' => 'Up ' . \round($executionEnd - $executionStart, 2) . 's',
- 'key' => $secret,
- ]);
-
- Console::success('Runtime Server created in ' . ($executionEnd - $executionStart) . ' seconds');
- } catch (\Throwable $th) {
- Console::error('Runtime Server Creation Failed: '. $th->getMessage());
- throw new Exception($th->getMessage(), 500);
- } finally {
- $orchestrationPool->put($orchestration);
- }
-
$response
->setStatusCode(Response::STATUS_CODE_CREATED)
- ->json($build);
+ ->json($container);
});
@@ -355,51 +304,47 @@ App::get('/v1/runtimes')
}
$response
- ->setStatusCode(200)
+ ->setStatusCode(Response::STATUS_CODE_OK)
->json($runtimes);
});
App::get('/v1/runtimes/:runtimeId')
->desc("Get a runtime by its ID")
- ->param('runtimeId', '', new Text(62), 'Runtime unique ID.')
+ ->param('runtimeId', '', new Text(64), 'Runtime unique ID.')
->inject('activeRuntimes')
->inject('response')
->action(function ($runtimeId, $activeRuntimes, Response $response) {
- $container = 'r-' . $runtimeId;
-
- if(!$activeRuntimes->exists($container)) {
+ if(!$activeRuntimes->exists($runtimeId)) {
throw new Exception('Runtime not found', 404);
}
- $runtime = $activeRuntimes->get($container);
+ $runtime = $activeRuntimes->get($runtimeId);
$response
- ->setStatusCode(200)
+ ->setStatusCode(Response::STATUS_CODE_OK)
->json($runtime);
});
App::delete('/v1/runtimes/:runtimeId')
->desc('Delete a runtime')
- ->param('runtimeId', '', new Text(62), 'Runtime unique ID.', false)
+ ->param('runtimeId', '', new Text(64), 'Runtime unique ID.', false)
->inject('orchestrationPool')
->inject('activeRuntimes')
->inject('response')
->action(function (string $runtimeId, $orchestrationPool, $activeRuntimes, Response $response) {
- $container = 'r-' . $runtimeId;
-
- if(!$activeRuntimes->exists($container)) {
+ if(!$activeRuntimes->exists($runtimeId)) {
throw new Exception('Runtime not found', 404);
}
- Console::info('Deleting runtime: ' . $container);
+ Console::info('Deleting runtime: ' . $runtimeId);
try {
$orchestration = $orchestrationPool->get();
- $orchestration->remove($container, true);
- $activeRuntimes->del($container);
- Console::success('Removed runtime container: ' . $container);
+ $orchestration->remove($runtimeId, true);
+ $activeRuntimes->del($runtimeId);
+ Console::success('Removed runtime container: ' . $runtimeId);
} finally {
$orchestrationPool->put($orchestration);
}
@@ -423,7 +368,7 @@ App::delete('/v1/runtimes/:runtimeId')
App::post('/v1/execution')
->desc('Create an execution')
- ->param('runtimeId', '', new Text(62), 'The runtimeID to execute')
+ ->param('runtimeId', '', new Text(64), 'The runtimeID to execute')
->param('path', '', new Text(0), 'Path containing the built files.', false)
->param('vars', [], new Assoc(), 'Environment variables required for the build', false)
->param('data', '', new Text(8192), 'Data to be forwarded to the function, this is user specified.', true)
@@ -436,13 +381,11 @@ App::post('/v1/execution')
->action(
function (string $runtimeId, string $path, array $vars, string $data, string $runtime, string $entrypoint, $timeout, string $baseImage, $activeRuntimes, Response $response) {
- $container = 'r-' . $runtimeId;
-
- if (!$activeRuntimes->exists($container)) {
+ if (!$activeRuntimes->exists($runtimeId)) {
throw new Exception('Runtime not found. Please create the runtime.', 404);
}
- $runtime = $activeRuntimes->get($container);
+ $runtime = $activeRuntimes->get($runtimeId);
$secret = $runtime['key'];
if (empty($secret)) {
throw new Exception('Runtime secret not found. Please re-create the runtime.', 500);
@@ -460,13 +403,11 @@ App::post('/v1/execution')
$ch = \curl_init();
$body = \json_encode([
- 'path' => '/usr/code',
- 'file' => $entrypoint,
'env' => $vars,
'payload' => $data,
'timeout' => $timeout ?? (int) App::getEnv('_APP_FUNCTIONS_TIMEOUT', 900)
]);
- \curl_setopt($ch, CURLOPT_URL, "http://" . $container . ":3000/");
+ \curl_setopt($ch, CURLOPT_URL, "http://" . $runtimeId . ":3000/");
\curl_setopt($ch, CURLOPT_POST, true);
\curl_setopt($ch, CURLOPT_POSTFIELDS, $body);
\curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
@@ -497,7 +438,7 @@ App::post('/v1/execution')
// 110 is the Swoole error code for timeout, see: https://www.swoole.co.uk/docs/swoole-error-code
if ($errNo !== 0 && $errNo !== CURLE_COULDNT_CONNECT && $errNo !== CURLE_OPERATION_TIMEDOUT && $errNo !== 110) {
- throw new Exception('An internal curl error has occurred within the executor! Error Msg: ' . $error, 500);
+ throw new Exception('An internal curl error has occurred within the executor! Error Msg: ' . $error, 406);
}
$executionData = [];
@@ -542,7 +483,7 @@ App::post('/v1/execution')
/** Update swoole table */
$runtime['updated'] = \time();
- $activeRuntimes->set($container, $runtime);
+ $activeRuntimes->set($runtimeId, $runtime);
$response
->setStatusCode(Response::STATUS_CODE_OK)
@@ -559,18 +500,20 @@ App::setResource('orchestrationPool', fn() => $orchestrationPool);
App::setResource('activeRuntimes', fn() => $activeRuntimes);
/** Set callbacks */
-App::error(function ($error, $response) {
- // $route = $utopia->match($request);
- // logError($error, "httpError", $route);
-
+App::error(function ($utopia, $error, $request, $response) {
+ $route = $utopia->match($request);
+ logError($error, "httpError", $route);
+
switch ($error->getCode()) {
case 400: // Error allowed publicly
case 401: // Error allowed publicly
case 402: // Error allowed publicly
case 403: // Error allowed publicly
case 404: // Error allowed publicly
+ case 406: // Error allowed publicly
case 409: // Error allowed publicly
case 412: // Error allowed publicly
+ case 425: // Error allowed publicly
case 429: // Error allowed publicly
case 501: // Error allowed publicly
case 503: // Error allowed publicly
@@ -586,7 +529,7 @@ App::error(function ($error, $response) {
'file' => $error->getFile(),
'line' => $error->getLine(),
'trace' => $error->getTrace(),
- 'version' => App::getEnv('OPENRUNTIMES_VERSION', 'UNKNOWN'),
+ 'version' => App::getEnv('_APP_VERSION', 'UNKNOWN')
];
$response
@@ -596,7 +539,7 @@ App::error(function ($error, $response) {
->setStatusCode($code);
$response->json($output);
-}, ['error', 'response']);
+}, ['utopia', 'error', 'request', 'response']);
App::init(function ($request, $response) {
$secretKey = $request->getHeader('x-appwrite-executor-key', '');
@@ -644,8 +587,7 @@ $http->on('start', function ($http) {
Console::info('Removing orphan runtimes...');
try {
$orchestration = $orchestrationPool->get();
- $orphans = $orchestration->list(['label' => 'openruntimes-type=function']);
- } catch (\Throwable $th) {
+ $orphans = $orchestration->list(['label' => 'openruntimes-type=runtime']);
} finally {
$orchestrationPool->put($orchestration);
}
@@ -715,7 +657,7 @@ $http->on('beforeShutdown', function() {
Console::info('Cleaning up containers before shutdown...');
$orchestration = $orchestrationPool->get();
- $functionsToRemove = $orchestration->list(['label' => 'openruntimes-type=function']);
+ $functionsToRemove = $orchestration->list(['label' => 'openruntimes-type=runtime']);
$orchestrationPool->put($orchestration);
foreach ($functionsToRemove as $container) {
@@ -742,7 +684,7 @@ $http->on('request', function (SwooleRequest $swooleRequest, SwooleResponse $swo
try {
$app->run($request, $response);
} catch (\Throwable $th) {
- // logError($e, "serverError");
+ logError($th, "serverError");
$swooleResponse->setStatusCode(500);
$output = [
'message' => 'Error: '. $th->getMessage(),
diff --git a/app/views/console/functions/function.phtml b/app/views/console/functions/function.phtml
index c4ef2a9374..f810741507 100644
--- a/app/views/console/functions/function.phtml
+++ b/app/views/console/functions/function.phtml
@@ -687,7 +687,7 @@ $usageStatsEnabled = $this->getParam('usageStatsEnabled', true);
(Max file size allowed: )
-
+