CodeMirror/appwrite
1
0
Fork 0
mirror of synced 2024-06-01 10:29:48 +12:00
Code Issues Projects Releases Wiki Activity

First Test

Browse source
This commit is contained in:
Bradley Schofield 2021-08-24 10:32:27 +01:00
parent 51e650a139
commit 82b7e18033
10 changed files with 840 additions and 274 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
CHANGES.md
View file

@ -1,3 +1,7 @@
# Unreleased Version 0.11.0
- Added ability to create syncronous function executions
- Introduced new execution model for functions
# Version 0.9.3
## Bugs

11
app/config/collections.php
View file

@ -1560,6 +1560,15 @@ $collections = [
'required' => false,
'array' => false,
],
[
'$collection' => Database::SYSTEM_COLLECTION_RULES,
'label' => 'Async',
'key' => 'async',
'type' => Database::SYSTEM_VAR_TYPE_NUMERIC,
'default' => '',
'required' => false,
'array' => false,
]
],
],
Database::SYSTEM_COLLECTION_TAGS => [
@ -1613,7 +1622,7 @@ $collections = [
'default' => '',
'required' => false,
'array' => false,
],
]
],
],
Database::SYSTEM_COLLECTION_EXECUTIONS => [

119
app/controllers/api/functions.php
View file

@ -22,6 +22,7 @@ use Utopia\Validator\WhiteList;
use Utopia\Config\Config;
use Cron\CronExpression;
use Utopia\Exception;
use Utopia\Validator\Boolean;
include_once __DIR__ . '/../shared/api.php';
@ -353,41 +354,31 @@ App::patch('/v1/functions/:functionId/tag')
/** @var Appwrite\Database\Database $projectDB */
/** @var Appwrite\Database\Document $project */
$function = $projectDB->getDocument($functionId);
$tag = $projectDB->getDocument($tag);
if (empty($function->getId()) || Database::SYSTEM_COLLECTION_FUNCTIONS != $function->getCollection()) {
throw new Exception('Function not found', 404);
}
if (empty($tag->getId()) || Database::SYSTEM_COLLECTION_TAGS != $tag->getCollection()) {
throw new Exception('Tag not found', 404);
}
$schedule = $function->getAttribute('schedule', '');
$cron = (empty($function->getAttribute('tag')) && !empty($schedule)) ? new CronExpression($schedule) : null;
$next = (empty($function->getAttribute('tag')) && !empty($schedule)) ? $cron->getNextRunDate()->format('U') : null;
$function = $projectDB->updateDocument(array_merge($function->getArrayCopy(), [
'tag' => $tag->getId(),
'scheduleNext' => $next,
$ch = \curl_init();
\curl_setopt($ch, CURLOPT_URL, "http://executor:8080/v1/tag");
\curl_setopt($ch, CURLOPT_POST, true);
\curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode([
'functionId' => $functionId,
'tagId' => $tag
]));
\curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
\curl_setopt($ch, CURLOPT_TIMEOUT, 10);
\curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 10);
\curl_setopt($ch, CURLOPT_HTTPHEADER, [
'Content-Type: application/json',
'X-Appwrite-Project: '.$project->getId(),
]);
if ($next) { // Init first schedule
ResqueScheduler::enqueueAt($next, 'v1-functions', 'FunctionsV1', [
'projectId' => $project->getId(),
'webhooks' => $project->getAttribute('webhooks', []),
'functionId' => $function->getId(),
'executionId' => null,
'trigger' => 'schedule',
]); // Async task rescheduale
$executorResponse = \curl_exec($ch);
$error = \curl_error($ch);
if (!empty($error)) {
throw new Exception('Curl error: ' . $error, 500);
}
if (false === $function) {
throw new Exception('Failed saving function to DB', 500);
}
\curl_close($ch);
$response->dynamic($function, Response::MODEL_FUNCTION);
$response->dynamic(new Document(json_decode($executorResponse, true)), Response::MODEL_EXECUTION);
});
App::delete('/v1/functions/:functionId')
@ -506,7 +497,7 @@ App::post('/v1/functions/:functionId/tags')
'dateCreated' => time(),
'command' => $command,
'path' => $path,
'size' => $size,
'size' => $size
]);
if (false === $tag) {
@ -684,12 +675,12 @@ App::post('/v1/functions/:functionId/executions')
->label('abuse-time', 60)
->param('functionId', '', new UID(), 'Function unique ID.')
->param('data', '', new Text(8192), 'String of custom data to send to function.', true)
// ->param('async', 1, new Range(0, 1), 'Execute code asynchronously. Pass 1 for true, 0 for false. Default value is 1.', true)
->param('async', 1, new Range(0, 1), 'Execute code asynchronously. Pass 1 for true, 0 for false. Default value is 1.', true)
->inject('response')
->inject('project')
->inject('projectDB')
->inject('user')
->action(function ($functionId, $data, /*$async,*/ $response, $project, $projectDB, $user) {
->action(function ($functionId, $data, $async, $response, $project, $projectDB, $user) {
/** @var Appwrite\Utopia\Response $response */
/** @var Appwrite\Database\Document $project */
/** @var Appwrite\Database\Database $projectDB */
@ -736,7 +727,7 @@ App::post('/v1/functions/:functionId/executions')
'exitCode' => 0,
'stdout' => '',
'stderr' => '',
'time' => 0,
'time' => 0
]);
Authorization::reset();
@ -766,21 +757,55 @@ App::post('/v1/functions/:functionId/executions')
}
}
Resque::enqueue('v1-functions', 'FunctionsV1', [
'projectId' => $project->getId(),
'webhooks' => $project->getAttribute('webhooks', []),
'functionId' => $function->getId(),
'executionId' => $execution->getId(),
'trigger' => 'http',
'data' => $data,
'userId' => $user->getId(),
'jwt' => $jwt,
]);
if ($async) {
Resque::enqueue('v1-functions', 'FunctionsV1', [
'projectId' => $project->getId(),
'webhooks' => $project->getAttribute('webhooks', []),
'functionId' => $function->getId(),
'executionId' => $execution->getId(),
'trigger' => 'http',
'data' => $data,
'userId' => $user->getId(),
'jwt' => $jwt
]);
$response
return $response
->setStatusCode(Response::STATUS_CODE_CREATED)
->dynamic($execution, Response::MODEL_EXECUTION)
;
->dynamic($execution, Response::MODEL_EXECUTION);
}
// Directly execute function.
$ch = \curl_init();
\curl_setopt($ch, CURLOPT_URL, "http://executor:8080/v1/execute");
\curl_setopt($ch, CURLOPT_POST, true);
\curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode([
'trigger' => 'http',
'projectId' => $project->getId(),
'executionId' => $execution->getId(),
'functionId' => $function->getId(),
'data' => $data,
'webhooks' => $project->getAttribute('webhooks', []),
'userId' => $user->getId(),
'jwt' => $jwt,
]));
\curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
\curl_setopt($ch, CURLOPT_TIMEOUT, App::getEnv('_APP_FUNCTIONS_TIMEOUT', 900) + 200); // + 200 for safety margin
\curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 10);
\curl_setopt($ch, CURLOPT_HTTPHEADER, [
'Content-Type: application/json',
]);
$responseExecute = \curl_exec($ch);
$error = \curl_error($ch);
if (!empty($error)) {
Console::error('Curl error: '.$error);
}
\curl_close($ch);
$response
->setStatusCode(Response::STATUS_CODE_CREATED)
->dynamic(new Document(json_decode($responseExecute, true)), Response::MODEL_SYNC_EXECUTION);
});
App::get('/v1/functions/:functionId/executions')

546
app/executor.php Normal file
View file

@ -0,0 +1,546 @@
<?php
require_once __DIR__ . '/../vendor/autoload.php';
use Appwrite\Database\Database;
use Appwrite\Database\Document;
use Appwrite\Database\Adapter\MySQL as MySQLAdapter;
use Appwrite\Database\Adapter\Redis as RedisAdapter;
use Appwrite\Database\Validator\Authorization;
use Appwrite\Database\Validator\UID;
use Appwrite\Event\Event;
use Appwrite\Utopia\Response\Model\Execution;
use Utopia\App;
use Utopia\Swoole\Request;
use Appwrite\Utopia\Response;
use Utopia\CLI\Console;
use Swoole\Http\Server;
use Swoole\Http\Request as SwooleRequest;
use Swoole\Http\Response as SwooleResponse;
use Utopia\Orchestration\Adapter\DockerAPI;
use Utopia\Orchestration\Orchestration;
use Utopia\Orchestration\Container;
use Utopia\Orchestration\Exception\Timeout as TimeoutException;
use Utopia\Config\Config;
use Utopia\Validator\ArrayList;
use Utopia\Validator\JSON;
use Utopia\Validator\Text;
require_once __DIR__ . '/workers.php';
$dockerUser = App::getEnv('DOCKERHUB_PULL_USERNAME', null);
$dockerPass = App::getEnv('DOCKERHUB_PULL_PASSWORD', null);
$dockerEmail = App::getEnv('DOCKERHUB_PULL_EMAIL', null);
$orchestration = new Orchestration(new DockerAPI($dockerUser, $dockerPass, $dockerEmail));
$runtimes = Config::getParam('runtimes');
// NOTE: Triggers wierd cURL and Swoole bug, Need to look into.
// Co\run(function() use ($runtimes, $orchestration) { // Warmup: make sure images are ready to run fast 🚀
// foreach($runtimes as $runtime) {
// go(function() use ($runtime, $orchestration) {
// Console::info('Warming up '.$runtime['name'].' '.$runtime['version'].' environment...');
// $response = $orchestration->pull($runtime['image']);
// if ($response) {
// Console::success("Successfully Warmed up {$runtime['name']} {$runtime['version']}!");
// } else {
// Console::error("Failed to Warmup {$runtime['name']} {$runtime['version']}!");
// }
// });
// }
// });
/**
* List function servers
*/
$stdout = '';
$stderr = '';
$executionStart = \microtime(true);
$response = $orchestration->list(['label' => 'appwrite-type=function']);
$list = [];
foreach ($response as $value) {
$list[$value->getName()] = $value;
}
$executionEnd = \microtime(true);
Console::info(count($list).' functions listed in ' . ($executionEnd - $executionStart) . ' seconds');
App::post('/v1/execute') // Define Route
->inject('request')
->param('trigger', '', new Text(1024))
->param('projectId', '', new Text(1024))
->param('executionId', '', new Text(1024), '', true)
->param('functionId', '', new Text(1024))
->param('event', '', new Text(1024), '', true)
->param('eventData', '', new Text(1024), '', true)
->param('data', '', new Text(1024), '', true)
->param('webhooks', [], new ArrayList(new JSON()), [], true)
->param('userId', '', new Text(1024), '', true)
->param('JWT', '', new Text(1024), '', true)
->inject('response')
->action(
function ($trigger, $projectId, $executionId, $functionId, $event, $eventData, $data, $webhooks, $userId, $JWT, $request, $response) {
try {
$data = execute($trigger, $projectId, $executionId, $functionId, $event, $eventData, $data, $webhooks, $userId, $JWT);
return $response->json($data);
} catch (Exception $e) {
return $response
->addHeader('Cache-Control', 'no-cache, no-store, must-revalidate')
->addHeader('Expires', '0')
->addHeader('Pragma', 'no-cache')
->json(['error' => $e->getMessage()]);
}
}
);
App::post('/v1/tag')
->param('functionId', '', new UID(), 'Function unique ID.')
->param('tagId', '', new UID(), 'Tag unique ID.')
->inject('response')
->inject('projectDB')
->inject('projectID')
->action(function ($functionId, $tagId, $response, $projectDB, $projectID) {
global $register;
// Create new Database Instance
// $projectDB = new Database();
// $projectDB->setAdapter(new RedisAdapter(new MySQLAdapter($db, $cache), $cache));
// $projectDB->setNamespace('app_' . $projectId);
// $projectDB->setMocks(Config::getParam('collections', []));
Authorization::disable();
$project = $projectDB->getDocument($projectID);
$function = $projectDB->getDocument($functionId);
$tag = $projectDB->getDocument($tagId);
Authorization::reset();
if (empty($function->getId()) || Database::SYSTEM_COLLECTION_FUNCTIONS != $function->getCollection()) {
throw new Exception('Function not found', 404);
}
if (empty($tag->getId()) || Database::SYSTEM_COLLECTION_TAGS != $tag->getCollection()) {
throw new Exception('Tag not found', 404);
}
$schedule = $function->getAttribute('schedule', '');
$cron = (empty($function->getAttribute('tag')) && !empty($schedule)) ? new CronExpression($schedule) : null;
$next = (empty($function->getAttribute('tag')) && !empty($schedule)) ? $cron->getNextRunDate()->format('U') : null;
Authorization::disable();
$function = $projectDB->updateDocument(array_merge($function->getArrayCopy(), [
'tag' => $tag->getId(),
'scheduleNext' => $next
]));
Authorization::reset();
if ($next) { // Init first schedule
ResqueScheduler::enqueueAt($next, 'v1-functions', 'FunctionsV1', [
'projectId' => $projectID,
'webhooks' => $project->getAttribute('webhooks', []),
'functionId' => $function->getId(),
'executionId' => null,
'trigger' => 'schedule',
]); // Async task rescheduale
}
if (false === $function) {
throw new Exception('Failed saving function to DB', 500);
}
$response->dynamic($function, Response::MODEL_FUNCTION);
});
App::get('/v1/healthz')
->inject('request')
->inject('response')
->action(
function ($request, $response) {
$response
->addHeader('Cache-Control', 'no-cache, no-store, must-revalidate')
->addHeader('Expires', '0')
->addHeader('Pragma', 'no-cache')
->json(['status' => 'online']);
}
);
function execute(string $trigger, string $projectId, string $executionId, string $functionId, string $event = '', string $eventData = '', string $data = '', array $webhooks = [], string $userId = '', string $jwt = ''): array
{
global $list;
global $orchestration;
global $runtimes;
global $register;
$db = $register->get('db');
$cache = $register->get('cache');
// Create new Database Instance
$database = new Database();
$database->setAdapter(new RedisAdapter(new MySQLAdapter($db, $cache), $cache));
$database->setNamespace('app_' . $projectId);
$database->setMocks(Config::getParam('collections', []));
// Grab Tag Document
Authorization::disable();
$function = $database->getDocument($functionId);
$tag = $database->getDocument($function->getAttribute('tag', ''));
Authorization::reset();
if ($tag->getAttribute('functionId') !== $function->getId()) {
throw new Exception('Tag not found', 404);
}
Authorization::disable();
// Grab execution document if exists
// It it doesn't exist, create a new one.
$execution = (!empty($executionId)) ? $database->getDocument($executionId) : $database->createDocument([
'$collection' => Database::SYSTEM_COLLECTION_EXECUTIONS,
'$permissions' => [
'read' => [],
'write' => [],
],
'dateCreated' => time(),
'functionId' => $function->getId(),
'trigger' => $trigger, // http / schedule / event
'status' => 'processing', // waiting / processing / completed / failed
'exitCode' => 0,
'stdout' => '',
'stderr' => '',
'time' => 0,
]);
if (false === $execution || ($execution instanceof Document && $execution->isEmpty())) {
throw new Exception('Failed to create or read execution');
}
Authorization::reset();
// Check if runtime is active
$runtime = (isset($runtimes[$function->getAttribute('runtime', '')]))
? $runtimes[$function->getAttribute('runtime', '')]
: null;
if (\is_null($runtime)) {
throw new Exception('Runtime "' . $function->getAttribute('runtime', '') . '" is not supported');
}
// Process environment variables
$vars = \array_merge($function->getAttribute('vars', []), [
'APPWRITE_FUNCTION_ID' => $function->getId(),
'APPWRITE_FUNCTION_NAME' => $function->getAttribute('name', ''),
'APPWRITE_FUNCTION_TAG' => $tag->getId(),
'APPWRITE_FUNCTION_TRIGGER' => $trigger,
'APPWRITE_FUNCTION_RUNTIME_NAME' => $runtime['name'],
'APPWRITE_FUNCTION_RUNTIME_VERSION' => $runtime['version'],
'APPWRITE_FUNCTION_EVENT' => $event,
'APPWRITE_FUNCTION_EVENT_DATA' => $eventData,
'APPWRITE_FUNCTION_DATA' => $data,
'APPWRITE_FUNCTION_USER_ID' => $userId,
'APPWRITE_FUNCTION_JWT' => $jwt,
'APPWRITE_FUNCTION_PROJECT_ID' => $projectId,
]);
$tagPath = $tag->getAttribute('path', '');
$tagPathTarget = '/tmp/project-' . $projectId . '/' . $tag->getId() . '/code.tar.gz';
$tagPathTargetDir = \pathinfo($tagPathTarget, PATHINFO_DIRNAME);
$container = 'appwrite-function-' . $tag->getId();
$command = \escapeshellcmd($tag->getAttribute('command', ''));
if (!\is_readable($tagPath)) {
throw new Exception('Code is not readable: ' . $tag->getAttribute('path', ''));
}
if (!\file_exists($tagPathTargetDir)) {
if (!\mkdir($tagPathTargetDir, 0755, true)) {
throw new Exception('Can\'t create directory ' . $tagPathTargetDir);
}
}
if (!\file_exists($tagPathTarget)) {
if (!\copy($tagPath, $tagPathTarget)) {
throw new Exception('Can\'t create temporary code file ' . $tagPathTarget);
}
}
// Check if container is already online
if (isset($list[$container]) && !(\substr($list[$container]->getStatus(), 0, 2) === 'Up')) { // Remove conatiner if not online
$stdout = '';
$stderr = '';
// If container is online then stop and remove it
try {
$orchestration->remove($container);
} catch (Exception $e) {
Console::warning('Failed to remove container: ' . $e->getMessage());
}
unset($list[$container]);
}
/**
* Limit CPU Usage - DONE
* Limit Memory Usage - DONE
* Limit Network Usage
* Limit Storage Usage (//--storage-opt size=120m \)
* Make sure no access to redis, mariadb, influxdb or other system services
* Make sure no access to NFS server / storage volumes
* Access Appwrite REST from internal network for improved performance
*/
if (!isset($list[$container])) { // Create contianer if not ready
$stdout = '';
$stderr = '';
$executionStart = \microtime(true);
$executionTime = \time();
$orchestration->setCpus(App::getEnv('_APP_FUNCTIONS_CPUS', '1'));
$orchestration->setMemory(App::getEnv('_APP_FUNCTIONS_MEMORY', '256'));
$orchestration->setSwap(App::getEnv('_APP_FUNCTIONS_MEMORY_SWAP', '256'));
foreach ($vars as &$value) {
$value = strval($value);
}
$id = $orchestration->run(
image: $runtime['image'],
name: $container,
command: [
'tail',
'-f',
'/dev/null'
],
entrypoint: '',
workdir: '/usr/local/src',
volumes: [],
vars: $vars,
mountFolder: $tagPathTargetDir,
labels: [
'appwrite-type' => 'function',
'appwrite-created' => strval($executionTime)
]
);
$untarStdout = '';
$untarStderr = '';
$untarSuccess = $orchestration->execute(
name: $container,
command: [
'sh',
'-c',
'mv /tmp/code.tar.gz /usr/local/src/code.tar.gz && tar -zxf /usr/local/src/code.tar.gz --strip 1 && rm /usr/local/src/code.tar.gz'
],
stdout: $untarStdout,
stderr: $untarStderr,
vars: $vars,
timeout: 60
);
if (!$untarSuccess) {
throw new Exception('Failed to extract tar: ' . $untarStderr);
}
$executionEnd = \microtime(true);
$list[$container] = new Container(
$container,
$id,
'Up',
[
'appwrite-type' => 'function',
'appwrite-created' => strval($executionTime),
]
);
Console::info('Function created in ' . ($executionEnd - $executionStart) . ' seconds');
} else {
Console::info('Container is ready to run');
}
$stdout = '';
$stderr = '';
$executionStart = \microtime(true);
$exitCode = 0;
// Execute function
try {
$exitCode = (int)!$orchestration->execute(
name: $container,
command: $orchestration->parseCommandString($command),
stdout: $stdout,
stderr: $stderr,
vars: $vars,
timeout: $function->getAttribute('timeout', (int) App::getEnv('_APP_FUNCTIONS_TIMEOUT', 900))
);
} catch (TimeoutException $e) {
$exitCode = 124;
}
$executionEnd = \microtime(true);
$executionTime = ($executionEnd - $executionStart);
$functionStatus = ($exitCode === 0) ? 'completed' : 'failed';
Console::info('Function executed in ' . ($executionEnd - $executionStart) . ' seconds, status: ' . $functionStatus);
Authorization::disable();
$execution = $database->updateDocument(array_merge($execution->getArrayCopy(), [
'tagId' => $tag->getId(),
'status' => $functionStatus,
'exitCode' => $exitCode,
'stdout' => \mb_substr($stdout, -4000), // log last 4000 chars output
'stderr' => \mb_substr($stderr, -4000), // log last 4000 chars output
'time' => $executionTime
]));
Authorization::reset();
if (false === $function) {
throw new Exception('Failed saving execution to DB', 500);
}
$executionModel = new Execution();
$executionUpdate = new Event('v1-webhooks', 'WebhooksV1');
$executionUpdate
->setParam('projectId', $projectId)
->setParam('userId', $userId)
->setParam('webhooks', $webhooks)
->setParam('event', 'functions.executions.update')
->setParam('eventData', $execution->getArrayCopy(array_keys($executionModel->getRules())));
$executionUpdate->trigger();
$usage = new Event('v1-usage', 'UsageV1');
$usage
->setParam('projectId', $projectId)
->setParam('functionId', $function->getId())
->setParam('functionExecution', 1)
->setParam('functionStatus', $functionStatus)
->setParam('functionExecutionTime', $executionTime * 1000) // ms
->setParam('networkRequestSize', 0)
->setParam('networkResponseSize', 0);
if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') {
$usage->trigger();
}
return [
'status' => $functionStatus,
'exitCode' => $exitCode,
'stdout' => $stdout,
'stderr' => $stderr,
'time' => $executionTime
];
}
App::setMode(App::MODE_TYPE_PRODUCTION); // Define Mode
$http = new Server("0.0.0.0", 8080);
$http->on('request', function (SwooleRequest $swooleRequest, SwooleResponse $swooleResponse) {
global $register;
$request = new Request($swooleRequest);
$response = new Response($swooleResponse);
$app = new App('UTC');
$db = $register->get('dbPool')->get();
$redis = $register->get('redisPool')->get();
App::setResource('db', function () use (&$db) {
return $db;
});
App::setResource('cache', function () use (&$redis) {
return $redis;
});
$projectId = $request->getHeader('x-appwrite-project', '');
App::setResource('projectDB', function($db, $cache) use ($projectId) {
$projectDB = new Database();
$projectDB->setAdapter(new RedisAdapter(new MySQLAdapter($db, $cache), $cache));
$projectDB->setNamespace('app_'.$projectId);
$projectDB->setMocks(Config::getParam('collections', []));
return $projectDB;
}, ['db', 'cache']);
App::error(function ($error, $utopia, $request, $response) {
/** @var Exception $error */
/** @var Utopia\App $utopia */
/** @var Utopia\Swoole\Request $request */
/** @var Appwrite\Utopia\Response $response */
if ($error instanceof PDOException) {
throw $error;
}
$route = $utopia->match($request);
Console::error('[Error] Timestamp: '.date('c', time()));
if($route) {
Console::error('[Error] Method: '.$route->getMethod());
Console::error('[Error] URL: '.$route->getURL());
}
Console::error('[Error] Type: '.get_class($error));
Console::error('[Error] Message: '.$error->getMessage());
Console::error('[Error] File: '.$error->getFile());
Console::error('[Error] Line: '.$error->getLine());
$version = App::getEnv('_APP_VERSION', 'UNKNOWN');
$code = $error->getCode();
$message = $error->getMessage();
//$_SERVER = []; // Reset before reporting to error log to avoid keys being compromised
$output = ((App::isDevelopment())) ? [
'message' => $error->getMessage(),
'code' => $error->getCode(),
'file' => $error->getFile(),
'line' => $error->getLine(),
'trace' => $error->getTrace(),
'version' => $version,
] : [
'message' => $message,
'code' => $code,
'version' => $version,
];
$response
->addHeader('Cache-Control', 'no-cache, no-store, must-revalidate')
->addHeader('Expires', '0')
->addHeader('Pragma', 'no-cache')
->setStatusCode($code)
;
$response->dynamic(new Document($output),
$utopia->isDevelopment() ? Response::MODEL_ERROR_DEV : Response::MODEL_ERROR);
}, ['error', 'utopia', 'request', 'response']);
App::setResource('projectID', function() use ($projectId) {
return $projectId;
});
try {
$app->run($request, $response);
} catch (Exception $e) {
Console::error('There\'s a problem with ' . $request->getURI());
$swooleResponse->end('500: Server Error');
}
});
$http->start();

243
app/workers/functions.php
View file

@ -296,229 +296,36 @@ class FunctionsV1 extends Worker
*/
public function execute(string $trigger, string $projectId, string $executionId, Database $database, Document $function, string $event = '', string $eventData = '', string $data = '', array $webhooks = [], string $userId = '', string $jwt = ''): void
{
global $list;
$runtimes = Config::getParam('runtimes');
Authorization::disable();
$tag = $database->getDocument($function->getAttribute('tag', ''));
Authorization::reset();
if($tag->getAttribute('functionId') !== $function->getId()) {
throw new Exception('Tag not found', 404);
}
Authorization::disable();
$execution = (!empty($executionId)) ? $database->getDocument($executionId) : $database->createDocument([
'$collection' => Database::SYSTEM_COLLECTION_EXECUTIONS,
'$permissions' => [
'read' => [],
'write' => [],
],
'dateCreated' => time(),
$ch = \curl_init();
\curl_setopt($ch, CURLOPT_URL, "http://executor:8080/v1/execute");
\curl_setopt($ch, CURLOPT_POST, true);
\curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode([
'trigger' => $trigger,
'projectId' => $projectId,
'executionId' => $executionId,
'functionId' => $function->getId(),
'trigger' => $trigger, // http / schedule / event
'status' => 'processing', // waiting / processing / completed / failed
'exitCode' => 0,
'stdout' => '',
'stderr' => '',
'time' => 0,
]);
if(false === $execution || ($execution instanceof Document && $execution->isEmpty())) {
throw new Exception('Failed to create or read execution');
}
Authorization::reset();
$runtime = (isset($runtimes[$function->getAttribute('runtime', '')]))
? $runtimes[$function->getAttribute('runtime', '')]
: null;
if(\is_null($runtime)) {
throw new Exception('Runtime "'.$function->getAttribute('runtime', '').'" is not supported');
}
$vars = \array_merge($function->getAttribute('vars', []), [
'APPWRITE_FUNCTION_ID' => $function->getId(),
'APPWRITE_FUNCTION_NAME' => $function->getAttribute('name', ''),
'APPWRITE_FUNCTION_TAG' => $tag->getId(),
'APPWRITE_FUNCTION_TRIGGER' => $trigger,
'APPWRITE_FUNCTION_RUNTIME_NAME' => $runtime['name'],
'APPWRITE_FUNCTION_RUNTIME_VERSION' => $runtime['version'],
'APPWRITE_FUNCTION_EVENT' => $event,
'APPWRITE_FUNCTION_EVENT_DATA' => $eventData,
'APPWRITE_FUNCTION_DATA' => $data,
'APPWRITE_FUNCTION_USER_ID' => $userId,
'APPWRITE_FUNCTION_JWT' => $jwt,
'APPWRITE_FUNCTION_PROJECT_ID' => $projectId,
]);
\array_walk($vars, function (&$value, $key) {
$key = $this->filterEnvKey($key);
$value = \escapeshellarg((empty($value)) ? '' : $value);
$value = "--env {$key}={$value}";
});
$tagPath = $tag->getAttribute('path', '');
$tagPathTarget = '/tmp/project-'.$projectId.'/'.$tag->getId().'/code.tar.gz';
$tagPathTargetDir = \pathinfo($tagPathTarget, PATHINFO_DIRNAME);
$container = 'appwrite-function-'.$tag->getId();
$command = \escapeshellcmd($tag->getAttribute('command', ''));
if(!\is_readable($tagPath)) {
throw new Exception('Code is not readable: '.$tag->getAttribute('path', ''));
}
if (!\file_exists($tagPathTargetDir)) {
if (!\mkdir($tagPathTargetDir, 0755, true)) {
throw new Exception('Can\'t create directory '.$tagPathTargetDir);
}
}
if (!\file_exists($tagPathTarget)) {
if(!\copy($tagPath, $tagPathTarget)) {
throw new Exception('Can\'t create temporary code file '.$tagPathTarget);
}
}
if(isset($list[$container]) && !$list[$container]['online']) { // Remove conatiner if not online
$stdout = '';
$stderr = '';
if(Console::execute("docker rm {$container}", '', $stdout, $stderr, 30) !== 0) {
throw new Exception('Failed to remove offline container: '.$stderr);
}
unset($list[$container]);
}
/**
* Limit CPU Usage - DONE
* Limit Memory Usage - DONE
* Limit Network Usage
* Limit Storage Usage (//--storage-opt size=120m \)
* Make sure no access to redis, mariadb, influxdb or other system services
* Make sure no access to NFS server / storage volumes
* Access Appwrite REST from internal network for improved performance
*/
if(!isset($list[$container])) { // Create contianer if not ready
$stdout = '';
$stderr = '';
$executionStart = \microtime(true);
$executionTime = \time();
$cpus = App::getEnv('_APP_FUNCTIONS_CPUS', '');
$memory = App::getEnv('_APP_FUNCTIONS_MEMORY', '');
$swap = App::getEnv('_APP_FUNCTIONS_MEMORY_SWAP', '');
$exitCode = Console::execute("docker run ".
" -d".
" --entrypoint=\"\"".
(empty($cpus) ? "" : (" --cpus=".$cpus)).
(empty($memory) ? "" : (" --memory=".$memory."m")).
(empty($swap) ? "" : (" --memory-swap=".$swap."m")).
" --name={$container}".
" --label appwrite-type=function".
" --label appwrite-created={$executionTime}".
" --volume {$tagPathTargetDir}:/tmp:rw".
" --workdir /usr/local/src".
" ".\implode(" ", $vars).
" {$runtime['image']}".
" tail -f /dev/null"
, '', $stdout, $stderr, 30);
if($exitCode !== 0) {
throw new Exception('Failed to create function environment: '.$stderr);
}
$exitCodeUntar = Console::execute("docker exec ".
$container.
" sh -c 'mv /tmp/code.tar.gz /usr/local/src/code.tar.gz && tar -zxf /usr/local/src/code.tar.gz --strip 1 && rm /usr/local/src/code.tar.gz'"
, '', $stdout, $stderr, 60);
if($exitCodeUntar !== 0) {
throw new Exception('Failed to extract tar: '.$stderr);
}
$executionEnd = \microtime(true);
$list[$container] = [
'name' => $container,
'online' => true,
'status' => 'Up',
'labels' => [
'appwrite-type' => 'function',
'appwrite-created' => $executionTime,
],
];
Console::info("Function created in " . ($executionEnd - $executionStart) . " seconds with exit code {$exitCode}");
}
else {
Console::info('Container is ready to run');
}
$stdout = '';
$stderr = '';
$executionStart = \microtime(true);
$exitCode = Console::execute("docker exec ".\implode(" ", $vars)." {$container} {$command}"
, '', $stdout, $stderr, $function->getAttribute('timeout', (int) App::getEnv('_APP_FUNCTIONS_TIMEOUT', 900)));
$executionEnd = \microtime(true);
$executionTime = ($executionEnd - $executionStart);
$functionStatus = ($exitCode === 0) ? 'completed' : 'failed';
Console::info("Function executed in " . ($executionEnd - $executionStart) . " seconds with exit code {$exitCode}");
Authorization::disable();
$execution = $database->updateDocument(array_merge($execution->getArrayCopy(), [
'tagId' => $tag->getId(),
'status' => $functionStatus,
'exitCode' => $exitCode,
'stdout' => \mb_substr($stdout, -4000), // log last 4000 chars output
'stderr' => \mb_substr($stderr, -4000), // log last 4000 chars output
'time' => $executionTime,
'event' => $event,
'eventData' => $eventData,
'data' => $data,
'webhooks' => $webhooks,
'userId' => $userId,
'jwt' => $jwt,
]));
Authorization::reset();
\curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
\curl_setopt($ch, CURLOPT_TIMEOUT, App::getEnv('_APP_FUNCTIONS_TIMEOUT', 900) + 200); // + 200 for safety margin
\curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 10);
\curl_setopt($ch, CURLOPT_HTTPHEADER, [
'Content-Type: application/json',
]);
if (false === $function) {
throw new Exception('Failed saving execution to DB', 500);
$response = \curl_exec($ch);
$error = \curl_error($ch);
if (!empty($error)) {
Console::error('Curl error: '.$error);
}
$executionModel = new Execution();
$executionUpdate = new Event('v1-webhooks', 'WebhooksV1');
$executionUpdate
->setParam('projectId', $projectId)
->setParam('userId', $userId)
->setParam('webhooks', $webhooks)
->setParam('event', 'functions.executions.update')
->setParam('eventData', $execution->getArrayCopy(array_keys($executionModel->getRules())));
$executionUpdate->trigger();
$usage = new Event('v1-usage', 'UsageV1');
$usage
->setParam('projectId', $projectId)
->setParam('functionId', $function->getId())
->setParam('functionExecution', 1)
->setParam('functionStatus', $functionStatus)
->setParam('functionExecutionTime', $executionTime * 1000) // ms
->setParam('networkRequestSize', 0)
->setParam('networkResponseSize', 0)
;
if(App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') {
$usage->trigger();
}
$this->cleanup();
\curl_close($ch);
}
/**

1
composer.json
View file

@ -52,6 +52,7 @@
"utopia-php/swoole": "0.2.*",
"utopia-php/storage": "0.5.*",
"utopia-php/image": "0.5.*",
"utopia-php/orchestration": "0.2.*",
"resque/php-resque": "1.3.6",
"matomo/device-detector": "4.2.3",
"dragonmantank/cron-expression": "3.1.0",

70
composer.lock generated
View file

@ -4,7 +4,7 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
"This file is @generated automatically"
],
"content-hash": "63a89a825697892a52aa27d6819b5972",
"content-hash": "aff105f689d2f08f13b6f3efef690d2c",
"packages": [
{
"name": "adhocore/jwt",
@ -1756,16 +1756,16 @@
},
{
"name": "utopia-php/framework",
"version": "0.17.2",
"version": "0.17.3",
"source": {
"type": "git",
"url": "https://github.com/utopia-php/framework.git",
"reference": "3cd5fa2a9e30040277861f4254c5ccd1b1600952"
"reference": "0274f6b3e49db2af0d702edf278ec7504dc99878"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/utopia-php/framework/zipball/3cd5fa2a9e30040277861f4254c5ccd1b1600952",
"reference": "3cd5fa2a9e30040277861f4254c5ccd1b1600952",
"url": "https://api.github.com/repos/utopia-php/framework/zipball/0274f6b3e49db2af0d702edf278ec7504dc99878",
"reference": "0274f6b3e49db2af0d702edf278ec7504dc99878",
"shasum": ""
},
"require": {
@ -1799,9 +1799,9 @@
],
"support": {
"issues": "https://github.com/utopia-php/framework/issues",
"source": "https://github.com/utopia-php/framework/tree/0.17.2"
"source": "https://github.com/utopia-php/framework/tree/0.17.3"
},
"time": "2021-08-02T10:18:26+00:00"
"time": "2021-08-03T13:57:01+00:00"
},
{
"name": "utopia-php/image",
@ -1907,6 +1907,61 @@
},
"time": "2021-07-24T11:35:55+00:00"
},
{
"name": "utopia-php/orchestration",
"version": "0.2.0",
"source": {
"type": "git",
"url": "https://github.com/utopia-php/orchestration.git",
"reference": "de10509017768cf2b62363bb39912002ab41dafb"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/utopia-php/orchestration/zipball/de10509017768cf2b62363bb39912002ab41dafb",
"reference": "de10509017768cf2b62363bb39912002ab41dafb",
"shasum": ""
},
"require": {
"php": ">=8.0",
"utopia-php/cli": "0.11.*"
},
"require-dev": {
"phpunit/phpunit": "^9.3",
"vimeo/psalm": "4.0.1"
},
"type": "library",
"autoload": {
"psr-4": {
"Utopia\\Orchestration\\": "src/Orchestration"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Eldad Fux",
"email": "eldad@appwrite.io"
}
],
"description": "Lite & fast micro PHP abstraction library for container orchestration",
"keywords": [
"docker",
"framework",
"kubernetes",
"orchestration",
"php",
"swarm",
"upf",
"utopia"
],
"support": {
"issues": "https://github.com/utopia-php/orchestration/issues",
"source": "https://github.com/utopia-php/orchestration/tree/0.2.0"
},
"time": "2021-08-16T12:52:42+00:00"
},
{
"name": "utopia-php/preloader",
"version": "0.2.4",
@ -4882,7 +4937,6 @@
"type": "github"
}
],
"abandoned": true,
"time": "2020-09-28T06:45:17+00:00"
},
{

52
docker-compose.yml
View file

@ -313,6 +313,57 @@ services:
- DOCKERHUB_PULL_USERNAME
- DOCKERHUB_PULL_PASSWORD
appwrite-executor:
entrypoint:
- php
- -e
- app/executor.php
- -dopcache.preload=opcache.preload=/usr/src/code/app/preload.php
container_name: appwrite-executor
ports:
- "8080:8080"
build:
context: .
args:
- DEBUG=true
- TESTING=true
- VERSION=dev
networks:
appwrite:
aliases:
- executor
volumes:
- /var/run/docker.sock:/var/run/docker.sock
- appwrite-functions:/storage/functions:rw
- /tmp:/tmp:rw
- ./app:/usr/src/code/app
- ./src:/usr/src/code/src
- ./dev:/usr/local/dev
depends_on:
- redis
- mariadb
environment:
- _APP_ENV
- _APP_OPENSSL_KEY_V1
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_DB_HOST
- _APP_DB_PORT
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_FUNCTIONS_TIMEOUT
- _APP_FUNCTIONS_CONTAINERS
- _APP_FUNCTIONS_RUNTIMES
- _APP_FUNCTIONS_CPUS
- _APP_FUNCTIONS_MEMORY
- _APP_FUNCTIONS_MEMORY_SWAP
- _APP_USAGE_STATS
- DOCKERHUB_PULL_USERNAME
- DOCKERHUB_PULL_PASSWORD
appwrite-worker-mails:
entrypoint: worker-mails
container_name: appwrite-worker-mails
@ -547,4 +598,5 @@ volumes:
appwrite-functions:
appwrite-influxdb:
appwrite-config:
appwrite-executor:
# appwrite-chronograf:

3
src/Appwrite/Utopia/Response.php
View file

@ -20,6 +20,7 @@ use Appwrite\Utopia\Response\Model\Domain;
use Appwrite\Utopia\Response\Model\Error;
use Appwrite\Utopia\Response\Model\ErrorDev;
use Appwrite\Utopia\Response\Model\Execution;
use Appwrite\Utopia\Response\Model\SyncExecution;
use Appwrite\Utopia\Response\Model\File;
use Appwrite\Utopia\Response\Model\Func;
use Appwrite\Utopia\Response\Model\JWT;
@ -105,6 +106,7 @@ class Response extends SwooleResponse
const MODEL_TAG = 'tag';
const MODEL_TAG_LIST = 'tagList';
const MODEL_EXECUTION = 'execution';
const MODEL_SYNC_EXECUTION = 'syncExecution';
const MODEL_EXECUTION_LIST = 'executionList';
// Project
@ -188,6 +190,7 @@ class Response extends SwooleResponse
->setModel(new Func())
->setModel(new Tag())
->setModel(new Execution())
->setModel(new SyncExecution())
->setModel(new Project())
->setModel(new Webhook())
->setModel(new Key())

65
src/Appwrite/Utopia/Response/Model/SyncExecution.php Normal file
View file

@ -0,0 +1,65 @@
<?php
namespace Appwrite\Utopia\Response\Model;
use Appwrite\Utopia\Response;
use Appwrite\Utopia\Response\Model;
class SyncExecution extends Model
{
public function __construct()
{
$this
->addRule('status', [
'type' => self::TYPE_STRING,
'description' => 'Execution Status.',
'default' => '',
'example' => '5e5ea5c16897e',
])
->addRule('exitCode', [
'type' => self::TYPE_INTEGER,
'description' => 'Execution Exit Code.',
'default' => 0,
'example' => 0,
])
->addRule('stdout', [
'type' => self::TYPE_STRING,
'description' => 'Execution Stdout.',
'default' => '',
'example' => 'Hello World!',
])
->addRule('stderr', [
'type' => self::TYPE_STRING,
'description' => 'Execution Stderr.',
'default' => '',
'example' => 'An error occoured: ....... example',
])
->addRule('time', [
'type' => self::TYPE_INTEGER,
'description' => 'Execution Time.',
'default' => 0,
'example' => 100,
])
;
}
/**
* Get Name
*
* @return string
*/
public function getName():string
{
return 'Syncronous Execution';
}
/**
* Get Collection
*
* @return string
*/
public function getType():string
{
return Response::MODEL_SYNC_EXECUTION;
}
}