2020-05-05 01:34:31 +12:00
|
|
|
<?php
|
2020-07-17 00:04:06 +12:00
|
|
|
ini_set('display_errors', 1);
|
|
|
|
ini_set('display_startup_errors', 1);
|
|
|
|
error_reporting(E_ALL);
|
|
|
|
|
|
|
|
use Appwrite\Database\Database;
|
|
|
|
use Appwrite\Database\Adapter\MySQL as MySQLAdapter;
|
|
|
|
use Appwrite\Database\Adapter\Redis as RedisAdapter;
|
|
|
|
use Appwrite\Database\Validator\Authorization;
|
2020-05-10 10:12:00 +12:00
|
|
|
use Utopia\CLI\Console;
|
2020-05-05 01:34:31 +12:00
|
|
|
use Utopia\Config\Config;
|
2020-05-09 18:26:18 +12:00
|
|
|
|
2020-05-10 04:39:50 +12:00
|
|
|
require_once __DIR__.'/../init.php';
|
|
|
|
|
|
|
|
cli_set_process_title('Functions V1 Worker');
|
|
|
|
|
|
|
|
Console::success(APP_NAME.' functions worker v1 has started');
|
|
|
|
|
2020-05-10 10:12:00 +12:00
|
|
|
$environments = Config::getParam('environments');
|
2020-05-09 18:26:18 +12:00
|
|
|
|
2020-07-16 00:49:10 +12:00
|
|
|
$warmupStart = microtime(true);
|
|
|
|
|
2020-07-16 01:34:28 +12:00
|
|
|
Co\run(function() use ($environments) {
|
|
|
|
foreach($environments as $environment) { // Warmup: make sure images are ready to run fast 🚀
|
|
|
|
go(function() use ($environment) {
|
|
|
|
$stdout = '';
|
|
|
|
$stderr = '';
|
|
|
|
|
|
|
|
Console::info('Warming up '.$environment['name'].' environment');
|
|
|
|
|
|
|
|
Console::execute('docker pull '.$environment['image'], null, $stdout, $stderr);
|
|
|
|
|
|
|
|
if(!empty($stdout)) {
|
|
|
|
Console::log($stdout);
|
|
|
|
}
|
|
|
|
|
|
|
|
if(!empty($stderr)) {
|
|
|
|
Console::error($stderr);
|
|
|
|
}
|
|
|
|
});
|
2020-05-10 10:12:00 +12:00
|
|
|
}
|
2020-07-16 01:34:28 +12:00
|
|
|
});
|
2020-05-05 01:34:31 +12:00
|
|
|
|
2020-07-16 00:49:10 +12:00
|
|
|
$warmupEnd = microtime(true);
|
|
|
|
$warmupTime = $warmupEnd - $warmupStart;
|
|
|
|
|
|
|
|
Console::success('Finished warmup in '.$warmupTime.' seconds');
|
|
|
|
|
2020-05-05 01:34:31 +12:00
|
|
|
class FunctionsV1
|
|
|
|
{
|
|
|
|
public $args = [];
|
|
|
|
|
|
|
|
public function setUp()
|
|
|
|
{
|
|
|
|
}
|
|
|
|
|
|
|
|
public function perform()
|
|
|
|
{
|
2020-07-17 00:04:06 +12:00
|
|
|
global $environments, $register;
|
2020-05-10 22:30:07 +12:00
|
|
|
|
2020-07-17 00:04:06 +12:00
|
|
|
$projectId = $this->args['projectId'];
|
2020-07-16 08:29:55 +12:00
|
|
|
$functionId = $this->args['functionId'];
|
|
|
|
$functionTag = $this->args['functionTag'];
|
|
|
|
|
2020-07-17 00:04:06 +12:00
|
|
|
$projectDB = new Database();
|
|
|
|
$projectDB->setAdapter(new RedisAdapter(new MySQLAdapter($register), $register));
|
|
|
|
$projectDB->setNamespace('app_'.$projectId);
|
|
|
|
$projectDB->setMocks(Config::getParam('collections', []));
|
|
|
|
|
|
|
|
Authorization::disable();
|
|
|
|
$function = $projectDB->getDocument($functionId);
|
|
|
|
Authorization::reset();
|
|
|
|
|
|
|
|
if (empty($function->getId()) || Database::SYSTEM_COLLECTION_FUNCTIONS != $function->getCollection()) {
|
|
|
|
throw new Exception('Function not found', 404);
|
|
|
|
}
|
|
|
|
|
|
|
|
Authorization::disable();
|
|
|
|
$tag = $projectDB->getDocument($functionTag);
|
|
|
|
Authorization::reset();
|
|
|
|
|
|
|
|
if($tag->getAttribute('functionId') !== $function->getId()) {
|
|
|
|
throw new Exception('Tag not found', 404);
|
|
|
|
}
|
|
|
|
|
|
|
|
$environment = (isset($environments[$function->getAttribute('env', '')]))
|
|
|
|
? $environments[$function->getAttribute('env', '')]
|
|
|
|
: null;
|
2020-07-16 08:29:55 +12:00
|
|
|
|
2020-07-17 00:04:06 +12:00
|
|
|
if(\is_null($environment)) {
|
|
|
|
throw new Exception('Environment "'.$function->getAttribute('env', '').' is not supported');
|
2020-07-16 08:29:55 +12:00
|
|
|
}
|
|
|
|
|
2020-05-13 10:00:48 +12:00
|
|
|
/*
|
|
|
|
* 1. Get Original Task
|
|
|
|
* 2. Check for updates
|
|
|
|
* If has updates skip task and don't reschedule
|
|
|
|
* If status not equal to play skip task
|
|
|
|
* 3. Check next run date, update task and add new job at the given date
|
|
|
|
* 4. Execute task (set optional timeout)
|
|
|
|
* 5. Update task response to log
|
|
|
|
* On success reset error count
|
|
|
|
* On failure add error count
|
|
|
|
* If error count bigger than allowed change status to pause
|
|
|
|
*/
|
|
|
|
|
2020-05-10 22:30:07 +12:00
|
|
|
/**
|
|
|
|
* 1. Get event args
|
2020-07-16 08:29:55 +12:00
|
|
|
* 2. Unpackage code in the isolated container
|
2020-05-13 10:00:48 +12:00
|
|
|
* 3. Execute in container with timeout
|
|
|
|
* + messure execution time
|
|
|
|
* + pass env vars
|
|
|
|
* + pass one-time api key
|
2020-05-10 22:30:07 +12:00
|
|
|
* 4. Update execution status
|
|
|
|
* 5. Update execution stdout & stderr
|
|
|
|
* 6. Trigger audit log
|
|
|
|
* 7. Trigger usage log
|
|
|
|
*/
|
2020-05-11 07:56:36 +12:00
|
|
|
$stdout = '';
|
|
|
|
$stderr = '';
|
|
|
|
$timeout = 15;
|
|
|
|
|
2020-07-17 00:04:06 +12:00
|
|
|
$start = \microtime(true);
|
2020-05-11 07:56:36 +12:00
|
|
|
|
2020-05-13 10:00:48 +12:00
|
|
|
//TODO aviod scheduled execution if delay is bigger than X offest
|
|
|
|
|
2020-05-11 18:48:26 +12:00
|
|
|
/**
|
2020-07-16 08:29:55 +12:00
|
|
|
* Limit CPU Usage - DONE
|
|
|
|
* Limit Memory Usage - DONE
|
2020-05-11 18:48:26 +12:00
|
|
|
* Limit Network Usage
|
|
|
|
* Make sure no access to redis, mariadb, influxdb or other system services
|
|
|
|
* Make sure no access to NFS server / storage volumes
|
|
|
|
* Access Appwrite REST from internal network for improved performance
|
|
|
|
*/
|
2020-07-17 00:04:06 +12:00
|
|
|
|
|
|
|
$tagPath = $tag->getAttribute('codePath', '');
|
|
|
|
$tagDir = \pathinfo($tag->getAttribute('codePath', ''), PATHINFO_DIRNAME);
|
|
|
|
$tagFile = \pathinfo($tag->getAttribute('codePath', ''), PATHINFO_BASENAME);
|
|
|
|
$tagPathTarget = '/tmp/project-'.$projectId.'/'.$tag->getId().'/code.tar.gz';
|
|
|
|
$tagPathTargetDir = \pathinfo($tagPathTarget, PATHINFO_DIRNAME);
|
|
|
|
|
|
|
|
if(!\is_readable($tagPath)) {
|
|
|
|
throw new Exception('Code is not readable: '.$tag->getAttribute('codePath', ''));
|
|
|
|
}
|
|
|
|
|
|
|
|
if (!\file_exists($tagPathTargetDir)) {
|
|
|
|
if (!\mkdir($tagPathTargetDir, 0755, true)) {
|
|
|
|
throw new Exception('Can\'t create directory '.$tagPathTargetDir);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (!\file_exists($tagPathTarget)) {
|
|
|
|
if(!\copy($tagPath, $tagPathTarget)) {
|
|
|
|
throw new Exception('Can\'t create temporary code file '.$tagPathTarget);
|
|
|
|
}
|
|
|
|
}
|
2020-07-16 08:29:55 +12:00
|
|
|
|
|
|
|
//--storage-opt size=120m \
|
|
|
|
$exitCode = Console::execute("docker run \
|
|
|
|
--cpus=1 \
|
|
|
|
--memory=50m \
|
|
|
|
--memory-swap=50m \
|
2020-05-11 07:56:36 +12:00
|
|
|
--rm \
|
2020-07-16 16:30:08 +12:00
|
|
|
--name=appwrite-function-{$functionId} \
|
2020-07-17 00:04:06 +12:00
|
|
|
--volume {$tagPathTargetDir}:/tmp:rw \
|
|
|
|
--workdir /usr/local/src \
|
2020-07-16 08:29:55 +12:00
|
|
|
--env APPWRITE_FUNCTION_ID={$functionId} \
|
|
|
|
--env APPWRITE_FUNCTION_TAG={$functionTag} \
|
|
|
|
--env APPWRITE_FUNCTION_ENV_NAME={$environment['name']} \
|
|
|
|
--env APPWRITE_FUNCTION_ENV_VERSION={$environment['version']} \
|
|
|
|
{$environment['image']} \
|
2020-07-17 00:04:06 +12:00
|
|
|
sh -c 'mv /tmp/code.tar.gz /usr/local/src/code.tar.gz && tar -zxf /usr/local/src/code.tar.gz --strip 1 && rm /usr/local/src/code.tar.gz && {$tag->getAttribute('command', '')}'"
|
|
|
|
, null, $stdout, $stderr, $timeout);
|
2020-05-10 22:30:07 +12:00
|
|
|
|
2020-07-17 00:04:06 +12:00
|
|
|
$end = \microtime(true);
|
|
|
|
|
|
|
|
var_dump('stdout', $stdout);
|
|
|
|
var_dump('stderr', $stderr);
|
2020-05-11 07:56:36 +12:00
|
|
|
|
2020-07-16 08:29:55 +12:00
|
|
|
Console::info("Code executed in " . ($end - $start) . " seconds with exit code {$exitCode}");
|
|
|
|
|
|
|
|
// Double-check Cleanup
|
2020-05-05 01:34:31 +12:00
|
|
|
}
|
|
|
|
|
|
|
|
public function tearDown()
|
|
|
|
{
|
|
|
|
}
|
|
|
|
}
|