Merge branch 'feat-functions-refactor' of github.com:appwrite/appwrite into feat-rename-tags
This commit is contained in:
commit
3685d8ef31
5 changed files with 351 additions and 466 deletions
|
@ -433,7 +433,7 @@ App::delete('/v1/functions/:functionId')
|
||||||
|
|
||||||
// Request executor to delete deployment containers
|
// Request executor to delete deployment containers
|
||||||
$ch = \curl_init();
|
$ch = \curl_init();
|
||||||
\curl_setopt($ch, CURLOPT_URL, "http://appwrite-executor:8080/v1/cleanup/function");
|
\curl_setopt($ch, CURLOPT_URL, "http://appwrite-executor/v1/cleanup/function");
|
||||||
\curl_setopt($ch, CURLOPT_POST, true);
|
\curl_setopt($ch, CURLOPT_POST, true);
|
||||||
\curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode([
|
\curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode([
|
||||||
'functionId' => $functionId
|
'functionId' => $functionId
|
||||||
|
@ -590,7 +590,7 @@ App::post('/v1/functions/:functionId/deployments')
|
||||||
$function = $dbForProject->getDocument('functions', $functionId);
|
$function = $dbForProject->getDocument('functions', $functionId);
|
||||||
|
|
||||||
$ch = \curl_init();
|
$ch = \curl_init();
|
||||||
\curl_setopt($ch, CURLOPT_URL, "http://appwrite-executor:8080/v1/deployment");
|
\curl_setopt($ch, CURLOPT_URL, "http://appwrite-executor/v1/deployment");
|
||||||
\curl_setopt($ch, CURLOPT_POST, true);
|
\curl_setopt($ch, CURLOPT_POST, true);
|
||||||
\curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode([
|
\curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode([
|
||||||
'functionId' => $function->getId(),
|
'functionId' => $function->getId(),
|
||||||
|
@ -771,7 +771,7 @@ App::delete('/v1/functions/:functionId/deployments/:deploymentId')
|
||||||
|
|
||||||
// Request executor to delete deployment containers
|
// Request executor to delete deployment containers
|
||||||
$ch = \curl_init();
|
$ch = \curl_init();
|
||||||
\curl_setopt($ch, CURLOPT_URL, "http://appwrite-executor:8080/v1/cleanup/deployment");
|
\curl_setopt($ch, CURLOPT_URL, "http://appwrite-executor/v1/cleanup/deployment");
|
||||||
\curl_setopt($ch, CURLOPT_POST, true);
|
\curl_setopt($ch, CURLOPT_POST, true);
|
||||||
\curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode([
|
\curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode([
|
||||||
'deploymentId' => $deploymentId
|
'deploymentId' => $deploymentId
|
||||||
|
@ -929,7 +929,7 @@ App::post('/v1/functions/:functionId/executions')
|
||||||
|
|
||||||
// Directly execute function.
|
// Directly execute function.
|
||||||
$ch = \curl_init();
|
$ch = \curl_init();
|
||||||
\curl_setopt($ch, CURLOPT_URL, "http://appwrite-executor:8080/v1/execute");
|
\curl_setopt($ch, CURLOPT_URL, "http://appwrite-executor/v1/execute");
|
||||||
\curl_setopt($ch, CURLOPT_POST, true);
|
\curl_setopt($ch, CURLOPT_POST, true);
|
||||||
\curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode([
|
\curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode([
|
||||||
'trigger' => 'http',
|
'trigger' => 'http',
|
||||||
|
@ -1161,7 +1161,7 @@ App::post('/v1/builds/:buildId')
|
||||||
|
|
||||||
// Retry build
|
// Retry build
|
||||||
$ch = \curl_init();
|
$ch = \curl_init();
|
||||||
\curl_setopt($ch, CURLOPT_URL, "http://appwrite-executor:8080/v1/build/{$buildId}");
|
\curl_setopt($ch, CURLOPT_URL, "http://appwrite-executor/v1/build/{$buildId}");
|
||||||
\curl_setopt($ch, CURLOPT_POST, true);
|
\curl_setopt($ch, CURLOPT_POST, true);
|
||||||
\curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
|
\curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
|
||||||
\curl_setopt($ch, CURLOPT_TIMEOUT, 900);
|
\curl_setopt($ch, CURLOPT_TIMEOUT, 900);
|
||||||
|
|
586
app/executor.php
586
app/executor.php
|
@ -1,40 +1,40 @@
|
||||||
<?php
|
<?php
|
||||||
require_once __DIR__ . '/../vendor/autoload.php';
|
require_once __DIR__ . '/../vendor/autoload.php';
|
||||||
|
|
||||||
use Utopia\Database\Document;
|
|
||||||
use Utopia\Database\Database;
|
|
||||||
use Utopia\Database\Validator\Authorization;
|
|
||||||
use Utopia\Database\Validator\UID;
|
|
||||||
use Appwrite\Event\Event;
|
use Appwrite\Event\Event;
|
||||||
use Appwrite\Utopia\Response\Model\Execution;
|
|
||||||
use Appwrite\Messaging\Adapter\Realtime;
|
use Appwrite\Messaging\Adapter\Realtime;
|
||||||
use Appwrite\Stats\Stats;
|
use Appwrite\Stats\Stats;
|
||||||
use Utopia\App;
|
|
||||||
use Utopia\Swoole\Request;
|
|
||||||
use Appwrite\Utopia\Response;
|
use Appwrite\Utopia\Response;
|
||||||
use Utopia\CLI\Console;
|
use Appwrite\Utopia\Response\Model\Execution;
|
||||||
use Swoole\Process;
|
use Cron\CronExpression;
|
||||||
use Swoole\Http\Server;
|
use Swoole\ConnectionPool;
|
||||||
|
use Swoole\Coroutine as Co;
|
||||||
use Swoole\Http\Request as SwooleRequest;
|
use Swoole\Http\Request as SwooleRequest;
|
||||||
use Swoole\Http\Response as SwooleResponse;
|
use Swoole\Http\Response as SwooleResponse;
|
||||||
use Utopia\Orchestration\Orchestration;
|
use Swoole\Http\Server;
|
||||||
use Utopia\Database\Adapter\MariaDB;
|
use Swoole\Process;
|
||||||
|
use Utopia\App;
|
||||||
|
use Utopia\CLI\Console;
|
||||||
use Utopia\Cache\Adapter\Redis as RedisCache;
|
use Utopia\Cache\Adapter\Redis as RedisCache;
|
||||||
|
use Utopia\Cache\Cache;
|
||||||
use Utopia\Config\Config;
|
use Utopia\Config\Config;
|
||||||
|
use Utopia\Database\Adapter\MariaDB;
|
||||||
|
use Utopia\Database\Database;
|
||||||
|
use Utopia\Database\Document;
|
||||||
|
use Utopia\Database\Query;
|
||||||
|
use Utopia\Database\Validator\Authorization;
|
||||||
|
use Utopia\Database\Validator\UID;
|
||||||
|
use Utopia\Logger\Log;
|
||||||
|
use Utopia\Orchestration\Adapter\DockerAPI;
|
||||||
|
use Utopia\Orchestration\Adapter\DockerCLI;
|
||||||
|
use Utopia\Orchestration\Orchestration;
|
||||||
|
use Utopia\Registry\Registry;
|
||||||
|
use Utopia\Storage\Device\Local;
|
||||||
|
use Utopia\Storage\Storage;
|
||||||
|
use Utopia\Swoole\Request;
|
||||||
use Utopia\Validator\ArrayList;
|
use Utopia\Validator\ArrayList;
|
||||||
use Utopia\Validator\JSON;
|
use Utopia\Validator\JSON;
|
||||||
use Utopia\Validator\Text;
|
use Utopia\Validator\Text;
|
||||||
use Cron\CronExpression;
|
|
||||||
use Swoole\ConnectionPool;
|
|
||||||
use Utopia\Storage\Device\Local;
|
|
||||||
use Utopia\Storage\Storage;
|
|
||||||
use Swoole\Coroutine as Co;
|
|
||||||
use Utopia\Cache\Cache;
|
|
||||||
use Utopia\Database\Query;
|
|
||||||
use Utopia\Orchestration\Adapter\DockerCLI;
|
|
||||||
use Utopia\Logger\Log;
|
|
||||||
use Utopia\Orchestration\Adapter\DockerAPI;
|
|
||||||
use Utopia\Registry\Registry;
|
|
||||||
|
|
||||||
require_once __DIR__ . '/init.php';
|
require_once __DIR__ . '/init.php';
|
||||||
|
|
||||||
|
@ -91,6 +91,7 @@ $orchestrationPool = new ConnectionPool(function () {
|
||||||
|
|
||||||
return $orchestration;
|
return $orchestration;
|
||||||
}, 6);
|
}, 6);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
$runtimes = Config::getParam('runtimes');
|
$runtimes = Config::getParam('runtimes');
|
||||||
|
|
||||||
|
@ -98,6 +99,7 @@ try {
|
||||||
Co\run(function () use ($runtimes, $orchestrationPool) {
|
Co\run(function () use ($runtimes, $orchestrationPool) {
|
||||||
foreach ($runtimes as $runtime) {
|
foreach ($runtimes as $runtime) {
|
||||||
go(function () use ($runtime, $orchestrationPool) {
|
go(function () use ($runtime, $orchestrationPool) {
|
||||||
|
try {
|
||||||
$orchestration = $orchestrationPool->get();
|
$orchestration = $orchestrationPool->get();
|
||||||
|
|
||||||
Console::info('Warming up ' . $runtime['name'] . ' ' . $runtime['version'] . ' environment...');
|
Console::info('Warming up ' . $runtime['name'] . ' ' . $runtime['version'] . ' environment...');
|
||||||
|
@ -109,8 +111,10 @@ try {
|
||||||
} else {
|
} else {
|
||||||
Console::warning("Failed to Warmup {$runtime['name']} {$runtime['version']}!");
|
Console::warning("Failed to Warmup {$runtime['name']} {$runtime['version']}!");
|
||||||
}
|
}
|
||||||
|
} catch (\Throwable $th) {
|
||||||
|
} finally {
|
||||||
$orchestrationPool->put($orchestration);
|
$orchestrationPool->put($orchestration);
|
||||||
|
}
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
@ -123,11 +127,15 @@ try {
|
||||||
$activeFunctions->create();
|
$activeFunctions->create();
|
||||||
|
|
||||||
Co\run(function () use ($orchestrationPool, $activeFunctions) {
|
Co\run(function () use ($orchestrationPool, $activeFunctions) {
|
||||||
|
try {
|
||||||
$orchestration = $orchestrationPool->get();
|
$orchestration = $orchestrationPool->get();
|
||||||
$executionStart = \microtime(true);
|
$executionStart = \microtime(true);
|
||||||
|
|
||||||
$residueList = $orchestration->list(['label' => 'appwrite-type=function']);
|
$residueList = $orchestration->list(['label' => 'appwrite-type=function']);
|
||||||
|
} catch (\Throwable $th) {
|
||||||
|
} finally {
|
||||||
$orchestrationPool->put($orchestration);
|
$orchestrationPool->put($orchestration);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
foreach ($residueList as $value) {
|
foreach ($residueList as $value) {
|
||||||
go(fn () => $activeFunctions->set($value->getName(), [
|
go(fn () => $activeFunctions->set($value->getName(), [
|
||||||
|
@ -311,9 +319,8 @@ function createRuntimeServer(string $functionId, string $projectId, string $depl
|
||||||
} catch (\Throwable $th) {
|
} catch (\Throwable $th) {
|
||||||
$orchestrationPool->put($orchestration);
|
$orchestrationPool->put($orchestration);
|
||||||
throw $th;
|
throw $th;
|
||||||
} finally {
|
|
||||||
$orchestrationPool->put($orchestration);
|
|
||||||
}
|
}
|
||||||
|
$orchestrationPool->put($orchestration);
|
||||||
};
|
};
|
||||||
|
|
||||||
function execute(string $trigger, string $projectId, string $executionId, string $functionId, Database $database, string $event = '', string $eventData = '', string $data = '', array $webhooks = [], string $userId = '', string $jwt = ''): array
|
function execute(string $trigger, string $projectId, string $executionId, string $functionId, Database $database, string $event = '', string $eventData = '', string $data = '', array $webhooks = [], string $userId = '', string $jwt = ''): array
|
||||||
|
@ -404,7 +411,7 @@ function execute(string $trigger, string $projectId, string $executionId, string
|
||||||
$database->createDocument('builds', new Document([
|
$database->createDocument('builds', new Document([
|
||||||
'$id' => $buildId,
|
'$id' => $buildId,
|
||||||
'$read' => ($userId !== '') ? ['user:' . $userId] : [],
|
'$read' => ($userId !== '') ? ['user:' . $userId] : [],
|
||||||
'$write' => ['role:all'],
|
'$write' => [],
|
||||||
'dateCreated' => time(),
|
'dateCreated' => time(),
|
||||||
'status' => 'processing',
|
'status' => 'processing',
|
||||||
'outputPath' => '',
|
'outputPath' => '',
|
||||||
|
@ -443,7 +450,7 @@ function execute(string $trigger, string $projectId, string $executionId, string
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
if (!$activeFunctions->exists($container)) { // Create contianer if not ready
|
if (!$activeFunctions->exists($container)) { // Create container if not ready
|
||||||
createRuntimeServer($functionId, $projectId, $deployment->getId(), $database);
|
createRuntimeServer($functionId, $projectId, $deployment->getId(), $database);
|
||||||
} else if ($activeFunctions->get($container)['status'] === 'Down') {
|
} else if ($activeFunctions->get($container)['status'] === 'Down') {
|
||||||
sleep(1);
|
sleep(1);
|
||||||
|
@ -651,6 +658,261 @@ function execute(string $trigger, string $projectId, string $executionId, string
|
||||||
];
|
];
|
||||||
};
|
};
|
||||||
|
|
||||||
|
function runBuildStage(string $buildId, string $projectID): Document
|
||||||
|
{
|
||||||
|
global $runtimes;
|
||||||
|
global $orchestrationPool;
|
||||||
|
global $register;
|
||||||
|
|
||||||
|
/** @var Orchestration $orchestration */
|
||||||
|
$orchestration = $orchestrationPool->get();
|
||||||
|
|
||||||
|
$buildStdout = '';
|
||||||
|
$buildStderr = '';
|
||||||
|
|
||||||
|
$db = $register->get('dbPool')->get();
|
||||||
|
$redis = $register->get('redisPool')->get();
|
||||||
|
$cache = new Cache(new RedisCache($redis));
|
||||||
|
|
||||||
|
$database = new Database(new MariaDB($db), $cache);
|
||||||
|
$database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite'));
|
||||||
|
$database->setNamespace('_project_' . $projectID);
|
||||||
|
|
||||||
|
// Check if build has already been run
|
||||||
|
$build = $database->getDocument('builds', $buildId);
|
||||||
|
|
||||||
|
try {
|
||||||
|
// If we already have a built package ready there is no need to rebuild.
|
||||||
|
if ($build->getAttribute('status') === 'ready' && \file_exists($build->getAttribute('outputPath'))) {
|
||||||
|
return $build;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Update Tag Status
|
||||||
|
$build->setAttribute('status', 'building');
|
||||||
|
|
||||||
|
$database->updateDocument('builds', $build->getId(), $build);
|
||||||
|
|
||||||
|
// Check if runtime is active
|
||||||
|
$runtime = $runtimes[$build->getAttribute('runtime', '')] ?? null;
|
||||||
|
|
||||||
|
if (\is_null($runtime)) {
|
||||||
|
throw new Exception('Runtime "' . $build->getAttribute('runtime', '') . '" is not supported');
|
||||||
|
}
|
||||||
|
|
||||||
|
// Grab Tag Files
|
||||||
|
$deploymentPath = $build->getAttribute('source', '');
|
||||||
|
$sourceType = $build->getAttribute('sourceType', '');
|
||||||
|
|
||||||
|
$device = Storage::getDevice('builds');
|
||||||
|
|
||||||
|
$deploymentPathTarget = '/tmp/project-' . $projectID . '/' . $build->getId() . '/code.tar.gz';
|
||||||
|
$deploymentPathTargetDir = \pathinfo($deploymentPathTarget, PATHINFO_DIRNAME);
|
||||||
|
|
||||||
|
$container = 'build-stage-' . $build->getId();
|
||||||
|
|
||||||
|
// Perform various checks
|
||||||
|
if (!\file_exists($deploymentPathTargetDir)) {
|
||||||
|
if (@\mkdir($deploymentPathTargetDir, 0777, true)) {
|
||||||
|
\chmod($deploymentPathTargetDir, 0777);
|
||||||
|
} else {
|
||||||
|
throw new Exception('Can\'t create directory ' . $deploymentPathTargetDir);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!\file_exists($deploymentPathTarget)) {
|
||||||
|
if (App::getEnv('_APP_STORAGE_DEVICE', Storage::DEVICE_LOCAL) === Storage::DEVICE_LOCAL) {
|
||||||
|
if (!\copy($deploymentPath, $deploymentPathTarget)) {
|
||||||
|
throw new Exception('Can\'t create temporary code file ' . $deploymentPathTarget);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
$buffer = $device->read($deploymentPath);
|
||||||
|
\file_put_contents($deploymentPathTarget, $buffer);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!$device->exists($deploymentPath)) {
|
||||||
|
throw new Exception('Code is not readable: ' . $build->getAttribute('source', ''));
|
||||||
|
}
|
||||||
|
|
||||||
|
$vars = $build->getAttribute('vars', []);
|
||||||
|
|
||||||
|
// Start tracking time
|
||||||
|
$buildStart = \microtime(true);
|
||||||
|
$time = \time();
|
||||||
|
|
||||||
|
$orchestration
|
||||||
|
->setCpus(App::getEnv('_APP_FUNCTIONS_CPUS', 0))
|
||||||
|
->setMemory(App::getEnv('_APP_FUNCTIONS_MEMORY', 256))
|
||||||
|
->setSwap(App::getEnv('_APP_FUNCTIONS_MEMORY_SWAP', 256));
|
||||||
|
|
||||||
|
$vars = array_map(fn ($v) => strval($v), $vars);
|
||||||
|
$path = '/tmp/project-' . $projectID . '/' . $build->getId() . '/builtCode';
|
||||||
|
|
||||||
|
if (!\file_exists($path)) {
|
||||||
|
if (@\mkdir($path, 0777, true)) {
|
||||||
|
\chmod($path, 0777);
|
||||||
|
} else {
|
||||||
|
throw new Exception('Can\'t create directory /tmp/project-' . $projectID . '/' . $build->getId() . '/builtCode');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Launch build container
|
||||||
|
$id = $orchestration->run(
|
||||||
|
image: $runtime['base'],
|
||||||
|
name: $container,
|
||||||
|
vars: $vars,
|
||||||
|
workdir: '/usr/code',
|
||||||
|
labels: [
|
||||||
|
'appwrite-type' => 'function',
|
||||||
|
'appwrite-created' => strval($time),
|
||||||
|
'appwrite-runtime' => $build->getAttribute('runtime', ''),
|
||||||
|
'appwrite-project' => $projectID,
|
||||||
|
'appwrite-build' => $build->getId(),
|
||||||
|
],
|
||||||
|
command: [
|
||||||
|
'tail',
|
||||||
|
'-f',
|
||||||
|
'/dev/null'
|
||||||
|
],
|
||||||
|
hostname: $container,
|
||||||
|
mountFolder: $deploymentPathTargetDir,
|
||||||
|
volumes: [
|
||||||
|
'/tmp/project-' . $projectID . '/' . $build->getId() . '/builtCode' . ':/usr/builtCode:rw'
|
||||||
|
]
|
||||||
|
);
|
||||||
|
|
||||||
|
if (empty($id)) {
|
||||||
|
throw new Exception('Failed to start build container');
|
||||||
|
}
|
||||||
|
|
||||||
|
// Extract user code into build container
|
||||||
|
$untarStdout = '';
|
||||||
|
$untarStderr = '';
|
||||||
|
|
||||||
|
$untarSuccess = $orchestration->execute(
|
||||||
|
name: $container,
|
||||||
|
command: [
|
||||||
|
'sh',
|
||||||
|
'-c',
|
||||||
|
'mkdir -p /usr/code && cp /tmp/code.tar.gz /usr/workspace/code.tar.gz && cd /usr/workspace/ && tar -zxf /usr/workspace/code.tar.gz -C /usr/code && rm /usr/workspace/code.tar.gz'
|
||||||
|
],
|
||||||
|
stdout: $untarStdout,
|
||||||
|
stderr: $untarStderr,
|
||||||
|
timeout: 60
|
||||||
|
);
|
||||||
|
|
||||||
|
if (!$untarSuccess) {
|
||||||
|
throw new Exception('Failed to extract tar: ' . $untarStderr);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Build Code / Install Dependencies
|
||||||
|
$buildSuccess = $orchestration->execute(
|
||||||
|
name: $container,
|
||||||
|
command: ['sh', '-c', 'cd /usr/local/src && ./build.sh'],
|
||||||
|
stdout: $buildStdout,
|
||||||
|
stderr: $buildStderr,
|
||||||
|
timeout: App::getEnv('_APP_FUNCTIONS_BUILD_TIMEOUT', 900)
|
||||||
|
);
|
||||||
|
|
||||||
|
if (!$buildSuccess) {
|
||||||
|
throw new Exception('Failed to build dependencies: ' . $buildStderr);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Repackage Code and Save.
|
||||||
|
$compressStdout = '';
|
||||||
|
$compressStderr = '';
|
||||||
|
|
||||||
|
$builtCodePath = '/tmp/project-' . $projectID . '/' . $build->getId() . '/builtCode/code.tar.gz';
|
||||||
|
|
||||||
|
$compressSuccess = $orchestration->execute(
|
||||||
|
name: $container,
|
||||||
|
command: [
|
||||||
|
'tar', '-C', '/usr/code', '-czvf', '/usr/builtCode/code.tar.gz', './'
|
||||||
|
],
|
||||||
|
stdout: $compressStdout,
|
||||||
|
stderr: $compressStderr,
|
||||||
|
timeout: 60
|
||||||
|
);
|
||||||
|
|
||||||
|
if (!$compressSuccess) {
|
||||||
|
throw new Exception('Failed to compress built code: ' . $compressStderr);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Remove Container
|
||||||
|
$orchestration->remove($id, true);
|
||||||
|
|
||||||
|
// Check if the build was successful by checking if file exists
|
||||||
|
if (!\file_exists($builtCodePath)) {
|
||||||
|
throw new Exception('Something went wrong during the build process.');
|
||||||
|
}
|
||||||
|
|
||||||
|
// Upload new code
|
||||||
|
$device = Storage::getDevice('builds');
|
||||||
|
|
||||||
|
$path = $device->getPath(\uniqid() . '.' . \pathinfo('code.tar.gz', PATHINFO_EXTENSION));
|
||||||
|
|
||||||
|
if (!\file_exists(\dirname($path))) { // Checks if directory path to file exists
|
||||||
|
if (@\mkdir(\dirname($path), 0777, true)) {
|
||||||
|
\chmod(\dirname($path), 0777);
|
||||||
|
} else {
|
||||||
|
throw new Exception('Can\'t create directory: ' . \dirname($path));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (App::getEnv('_APP_STORAGE_DEVICE', Storage::DEVICE_LOCAL) === Storage::DEVICE_LOCAL) {
|
||||||
|
if (!$device->move($builtCodePath, $path)) {
|
||||||
|
throw new Exception('Failed to upload built code upload to storage', 500);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
if (!$device->upload($builtCodePath, $path)) {
|
||||||
|
throw new Exception('Failed to upload built code upload to storage', 500);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if ($buildStdout == '') {
|
||||||
|
$buildStdout = 'Build Successful!';
|
||||||
|
}
|
||||||
|
|
||||||
|
$build
|
||||||
|
->setAttribute('outputPath', $path)
|
||||||
|
->setAttribute('status', 'ready')
|
||||||
|
->setAttribute('stdout', \utf8_encode(\mb_substr($buildStdout, -4096)))
|
||||||
|
->setAttribute('stderr', \utf8_encode(\mb_substr($buildStderr, -4096)))
|
||||||
|
->setAttribute('time', $time);
|
||||||
|
|
||||||
|
// Update build with built code attribute
|
||||||
|
$build = $database->updateDocument('builds', $buildId, $build);
|
||||||
|
|
||||||
|
$buildEnd = \microtime(true);
|
||||||
|
|
||||||
|
Console::info('Build Stage Ran in ' . ($buildEnd - $buildStart) . ' seconds');
|
||||||
|
} catch (Exception $e) {
|
||||||
|
$build
|
||||||
|
->setAttribute('status', 'failed')
|
||||||
|
->setAttribute('stdout', \utf8_encode(\mb_substr($buildStdout, -4096)))
|
||||||
|
->setAttribute('stderr', \utf8_encode(\mb_substr($e->getMessage(), -4096)));
|
||||||
|
|
||||||
|
$build = $database->updateDocument('builds', $buildId, $build);
|
||||||
|
|
||||||
|
// also remove the container if it exists
|
||||||
|
if (isset($id)) {
|
||||||
|
$orchestration->remove($id, true);
|
||||||
|
}
|
||||||
|
|
||||||
|
$register->get('dbPool')->put($db);
|
||||||
|
$register->get('redisPool')->put($redis);
|
||||||
|
|
||||||
|
throw new Exception('Build failed: ' . $e->getMessage());
|
||||||
|
}
|
||||||
|
|
||||||
|
$orchestrationPool->put($orchestration);
|
||||||
|
|
||||||
|
$register->get('dbPool')->put($db);
|
||||||
|
$register->get('redisPool')->put($redis);
|
||||||
|
|
||||||
|
return $build;
|
||||||
|
}
|
||||||
|
|
||||||
App::post('/v1/execute') // Define Route
|
App::post('/v1/execute') // Define Route
|
||||||
->desc('Execute a function')
|
->desc('Execute a function')
|
||||||
->param('trigger', '', new Text(1024))
|
->param('trigger', '', new Text(1024))
|
||||||
|
@ -682,7 +944,6 @@ App::post('/v1/execute') // Define Route
|
||||||
}
|
}
|
||||||
);
|
);
|
||||||
|
|
||||||
|
|
||||||
// Cleanup Endpoints used internally by appwrite when a function or deployment gets deleted to also clean up their containers
|
// Cleanup Endpoints used internally by appwrite when a function or deployment gets deleted to also clean up their containers
|
||||||
App::post('/v1/cleanup/function')
|
App::post('/v1/cleanup/function')
|
||||||
->param('functionId', '', new UID())
|
->param('functionId', '', new UID())
|
||||||
|
@ -857,6 +1118,7 @@ App::post('/v1/deployment')
|
||||||
|
|
||||||
// Build Code
|
// Build Code
|
||||||
go(function () use ($projectID, $deploymentId, $buildId, $functionId, $function, $register) {
|
go(function () use ($projectID, $deploymentId, $buildId, $functionId, $function, $register) {
|
||||||
|
try {
|
||||||
$db = $register->get('dbPool')->get();
|
$db = $register->get('dbPool')->get();
|
||||||
$redis = $register->get('redisPool')->get();
|
$redis = $register->get('redisPool')->get();
|
||||||
$cache = new Cache(new RedisCache($redis));
|
$cache = new Cache(new RedisCache($redis));
|
||||||
|
@ -883,7 +1145,7 @@ App::post('/v1/deployment')
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
if ($deployment->getAttribute('deploy') === true) {
|
if ($deployment->getAttribute('automaticDeploy') === true) {
|
||||||
// Update the function document setting the deployment as the active one
|
// Update the function document setting the deployment as the active one
|
||||||
$function
|
$function
|
||||||
->setAttribute('deployment', $deployment->getId())
|
->setAttribute('deployment', $deployment->getId())
|
||||||
|
@ -893,9 +1155,11 @@ App::post('/v1/deployment')
|
||||||
|
|
||||||
// Deploy Runtime Server
|
// Deploy Runtime Server
|
||||||
createRuntimeServer($functionId, $projectID, $deploymentId, $dbForProject);
|
createRuntimeServer($functionId, $projectID, $deploymentId, $dbForProject);
|
||||||
|
} catch (\Throwable $th) {
|
||||||
|
} finally {
|
||||||
$register->get('dbPool')->put($db);
|
$register->get('dbPool')->put($db);
|
||||||
$register->get('redisPool')->put($redis);
|
$register->get('redisPool')->put($redis);
|
||||||
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
if (false === $function) {
|
if (false === $function) {
|
||||||
|
@ -961,265 +1225,9 @@ App::post('/v1/build/:buildId') // Start a Build
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
function runBuildStage(string $buildId, string $projectID): Document
|
|
||||||
{
|
|
||||||
global $runtimes;
|
|
||||||
global $orchestrationPool;
|
|
||||||
global $register;
|
|
||||||
|
|
||||||
/** @var Orchestration $orchestration */
|
|
||||||
$orchestration = $orchestrationPool->get();
|
|
||||||
|
|
||||||
$buildStdout = '';
|
|
||||||
$buildStderr = '';
|
|
||||||
|
|
||||||
$db = $register->get('dbPool')->get();
|
|
||||||
$redis = $register->get('redisPool')->get();
|
|
||||||
$cache = new Cache(new RedisCache($redis));
|
|
||||||
|
|
||||||
$database = new Database(new MariaDB($db), $cache);
|
|
||||||
$database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite'));
|
|
||||||
$database->setNamespace('_project_' . $projectID);
|
|
||||||
|
|
||||||
// Check if build has already been run
|
|
||||||
$build = $database->getDocument('builds', $buildId);
|
|
||||||
|
|
||||||
try {
|
|
||||||
// If we already have a built package ready there is no need to rebuild.
|
|
||||||
if ($build->getAttribute('status') === 'ready' && \file_exists($build->getAttribute('outputPath'))) {
|
|
||||||
return $build;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Update Deployment Status
|
|
||||||
$build->setAttribute('status', 'building');
|
|
||||||
|
|
||||||
$database->updateDocument('builds', $build->getId(), $build);
|
|
||||||
|
|
||||||
// Check if runtime is active
|
|
||||||
$runtime = $runtimes[$build->getAttribute('runtime', '')] ?? null;
|
|
||||||
|
|
||||||
if (\is_null($runtime)) {
|
|
||||||
throw new Exception('Runtime "' . $build->getAttribute('runtime', '') . '" is not supported');
|
|
||||||
}
|
|
||||||
|
|
||||||
// Grab Deployment Files
|
|
||||||
$deploymentPath = $build->getAttribute('source', '');
|
|
||||||
$sourceType = $build->getAttribute('sourceType', '');
|
|
||||||
|
|
||||||
$device = Storage::getDevice('builds');
|
|
||||||
|
|
||||||
$deploymentPathTarget = '/tmp/project-' . $projectID . '/' . $build->getId() . '/code.tar.gz';
|
|
||||||
$deploymentPathTargetDir = \pathinfo($deploymentPathTarget, PATHINFO_DIRNAME);
|
|
||||||
|
|
||||||
$container = 'build-stage-' . $build->getId();
|
|
||||||
|
|
||||||
// Perform various checks
|
|
||||||
if (!\file_exists($deploymentPathTargetDir)) {
|
|
||||||
if (@\mkdir($deploymentPathTargetDir, 0777, true)) {
|
|
||||||
\chmod($deploymentPathTargetDir, 0777);
|
|
||||||
} else {
|
|
||||||
throw new Exception('Can\'t create directory ' . $deploymentPathTargetDir);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!\file_exists($deploymentPathTarget)) {
|
|
||||||
if (App::getEnv('_APP_STORAGE_DEVICE', Storage::DEVICE_LOCAL) === Storage::DEVICE_LOCAL) {
|
|
||||||
if (!\copy($deploymentPath, $deploymentPathTarget)) {
|
|
||||||
throw new Exception('Can\'t create temporary code file ' . $deploymentPathTarget);
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
$buffer = $device->read($deploymentPath);
|
|
||||||
\file_put_contents($deploymentPathTarget, $buffer);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!$device->exists($deploymentPath)) {
|
|
||||||
throw new Exception('Code is not readable: ' . $build->getAttribute('source', ''));
|
|
||||||
}
|
|
||||||
|
|
||||||
$vars = $build->getAttribute('vars', []);
|
|
||||||
|
|
||||||
// Start tracking time
|
|
||||||
$buildStart = \microtime(true);
|
|
||||||
$time = \time();
|
|
||||||
|
|
||||||
$orchestration
|
|
||||||
->setCpus(App::getEnv('_APP_FUNCTIONS_CPUS', 0))
|
|
||||||
->setMemory(App::getEnv('_APP_FUNCTIONS_MEMORY', 256))
|
|
||||||
->setSwap(App::getEnv('_APP_FUNCTIONS_MEMORY_SWAP', 256));
|
|
||||||
|
|
||||||
$vars = array_map(fn ($v) => strval($v), $vars);
|
|
||||||
$path = '/tmp/project-' . $projectID . '/' . $build->getId() . '/builtCode';
|
|
||||||
|
|
||||||
if (!\file_exists($path)) {
|
|
||||||
if (@\mkdir($path, 0777, true)) {
|
|
||||||
\chmod($path, 0777);
|
|
||||||
} else {
|
|
||||||
throw new Exception('Can\'t create directory /tmp/project-' . $projectID . '/' . $build->getId() . '/builtCode');
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
// Launch build container
|
|
||||||
$id = $orchestration->run(
|
|
||||||
image: $runtime['base'],
|
|
||||||
name: $container,
|
|
||||||
vars: $vars,
|
|
||||||
workdir: '/usr/code',
|
|
||||||
labels: [
|
|
||||||
'appwrite-type' => 'function',
|
|
||||||
'appwrite-created' => strval($time),
|
|
||||||
'appwrite-runtime' => $build->getAttribute('runtime', ''),
|
|
||||||
'appwrite-project' => $projectID,
|
|
||||||
'appwrite-build' => $build->getId(),
|
|
||||||
],
|
|
||||||
command: [
|
|
||||||
'tail',
|
|
||||||
'-f',
|
|
||||||
'/dev/null'
|
|
||||||
],
|
|
||||||
hostname: $container,
|
|
||||||
mountFolder: $deploymentPathTargetDir,
|
|
||||||
volumes: [
|
|
||||||
'/tmp/project-' . $projectID . '/' . $build->getId() . '/builtCode' . ':/usr/builtCode:rw'
|
|
||||||
]
|
|
||||||
);
|
|
||||||
|
|
||||||
if (empty($id)) {
|
|
||||||
throw new Exception('Failed to start build container');
|
|
||||||
}
|
|
||||||
|
|
||||||
// Extract user code into build container
|
|
||||||
$untarStdout = '';
|
|
||||||
$untarStderr = '';
|
|
||||||
|
|
||||||
$untarSuccess = $orchestration->execute(
|
|
||||||
name: $container,
|
|
||||||
command: [
|
|
||||||
'sh',
|
|
||||||
'-c',
|
|
||||||
'mkdir -p /usr/code && cp /tmp/code.tar.gz /usr/workspace/code.tar.gz && cd /usr/workspace/ && tar -zxf /usr/workspace/code.tar.gz -C /usr/code && rm /usr/workspace/code.tar.gz'
|
|
||||||
],
|
|
||||||
stdout: $untarStdout,
|
|
||||||
stderr: $untarStderr,
|
|
||||||
timeout: 60
|
|
||||||
);
|
|
||||||
|
|
||||||
if (!$untarSuccess) {
|
|
||||||
throw new Exception('Failed to extract tar: ' . $untarStderr);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Build Code / Install Dependencies
|
|
||||||
$buildSuccess = $orchestration->execute(
|
|
||||||
name: $container,
|
|
||||||
command: ['sh', '-c', 'cd /usr/local/src && ./build.sh'],
|
|
||||||
stdout: $buildStdout,
|
|
||||||
stderr: $buildStderr,
|
|
||||||
timeout: App::getEnv('_APP_FUNCTIONS_BUILD_TIMEOUT', 900)
|
|
||||||
);
|
|
||||||
|
|
||||||
if (!$buildSuccess) {
|
|
||||||
throw new Exception('Failed to build dependencies: ' . $buildStderr);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Repackage Code and Save.
|
|
||||||
$compressStdout = '';
|
|
||||||
$compressStderr = '';
|
|
||||||
|
|
||||||
$builtCodePath = '/tmp/project-' . $projectID . '/' . $build->getId() . '/builtCode/code.tar.gz';
|
|
||||||
|
|
||||||
$compressSuccess = $orchestration->execute(
|
|
||||||
name: $container,
|
|
||||||
command: [
|
|
||||||
'tar', '-C', '/usr/code', '-czvf', '/usr/builtCode/code.tar.gz', './'
|
|
||||||
],
|
|
||||||
stdout: $compressStdout,
|
|
||||||
stderr: $compressStderr,
|
|
||||||
timeout: 60
|
|
||||||
);
|
|
||||||
|
|
||||||
if (!$compressSuccess) {
|
|
||||||
throw new Exception('Failed to compress built code: ' . $compressStderr);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Remove Container
|
|
||||||
$orchestration->remove($id, true);
|
|
||||||
|
|
||||||
// Check if the build was successful by checking if file exists
|
|
||||||
if (!\file_exists($builtCodePath)) {
|
|
||||||
throw new Exception('Something went wrong during the build process.');
|
|
||||||
}
|
|
||||||
|
|
||||||
// Upload new code
|
|
||||||
$device = Storage::getDevice('builds');
|
|
||||||
|
|
||||||
$path = $device->getPath(\uniqid() . '.' . \pathinfo('code.tar.gz', PATHINFO_EXTENSION));
|
|
||||||
|
|
||||||
if (!\file_exists(\dirname($path))) { // Checks if directory path to file exists
|
|
||||||
if (@\mkdir(\dirname($path), 0777, true)) {
|
|
||||||
\chmod(\dirname($path), 0777);
|
|
||||||
} else {
|
|
||||||
throw new Exception('Can\'t create directory: ' . \dirname($path));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (App::getEnv('_APP_STORAGE_DEVICE', Storage::DEVICE_LOCAL) === Storage::DEVICE_LOCAL) {
|
|
||||||
if (!$device->move($builtCodePath, $path)) {
|
|
||||||
throw new Exception('Failed to upload built code upload to storage', 500);
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
if (!$device->upload($builtCodePath, $path)) {
|
|
||||||
throw new Exception('Failed to upload built code upload to storage', 500);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if ($buildStdout == '') {
|
|
||||||
$buildStdout = 'Build Successful!';
|
|
||||||
}
|
|
||||||
|
|
||||||
$build
|
|
||||||
->setAttribute('outputPath', $path)
|
|
||||||
->setAttribute('status', 'ready')
|
|
||||||
->setAttribute('stdout', \utf8_encode(\mb_substr($buildStdout, -4096)))
|
|
||||||
->setAttribute('stderr', \utf8_encode(\mb_substr($buildStderr, -4096)))
|
|
||||||
->setAttribute('time', $time);
|
|
||||||
|
|
||||||
// Update build with built code attribute
|
|
||||||
$build = $database->updateDocument('builds', $buildId, $build);
|
|
||||||
|
|
||||||
$buildEnd = \microtime(true);
|
|
||||||
|
|
||||||
Console::info('Build Stage Ran in ' . ($buildEnd - $buildStart) . ' seconds');
|
|
||||||
} catch (Exception $e) {
|
|
||||||
$build
|
|
||||||
->setAttribute('status', 'failed')
|
|
||||||
->setAttribute('stdout', \utf8_encode(\mb_substr($buildStdout, -4096)))
|
|
||||||
->setAttribute('stderr', \utf8_encode(\mb_substr($e->getMessage(), -4096)));
|
|
||||||
|
|
||||||
$build = $database->updateDocument('builds', $buildId, $build);
|
|
||||||
|
|
||||||
// also remove the container if it exists
|
|
||||||
if (isset($id)) {
|
|
||||||
$orchestration->remove($id, true);
|
|
||||||
}
|
|
||||||
|
|
||||||
$orchestrationPool->put($orchestration);
|
|
||||||
$register->get('dbPool')->put($db);
|
|
||||||
$register->get('redisPool')->put($redis);
|
|
||||||
|
|
||||||
throw new Exception('Build failed: ' . $e->getMessage());
|
|
||||||
} finally {
|
|
||||||
$orchestrationPool->put($orchestration);
|
|
||||||
|
|
||||||
$register->get('dbPool')->put($db);
|
|
||||||
$register->get('redisPool')->put($redis);
|
|
||||||
}
|
|
||||||
|
|
||||||
return $build;
|
|
||||||
}
|
|
||||||
|
|
||||||
App::setMode(App::MODE_TYPE_PRODUCTION); // Define Mode
|
App::setMode(App::MODE_TYPE_PRODUCTION); // Define Mode
|
||||||
|
|
||||||
$http = new Server("0.0.0.0", 8080);
|
$http = new Server("0.0.0.0", 80);
|
||||||
|
|
||||||
function handleShutdown()
|
function handleShutdown()
|
||||||
{
|
{
|
||||||
|
|
|
@ -194,10 +194,10 @@ services:
|
||||||
- _APP_USAGE_STATS
|
- _APP_USAGE_STATS
|
||||||
- _APP_STATSD_HOST
|
- _APP_STATSD_HOST
|
||||||
- _APP_STATSD_PORT
|
- _APP_STATSD_PORT
|
||||||
- DOCKERHUB_PULL_USERNAME
|
|
||||||
- DOCKERHUB_PULL_PASSWORD
|
|
||||||
- _APP_LOGGING_PROVIDER
|
- _APP_LOGGING_PROVIDER
|
||||||
- _APP_LOGGING_CONFIG
|
- _APP_LOGGING_CONFIG
|
||||||
|
- DOCKERHUB_PULL_USERNAME
|
||||||
|
- DOCKERHUB_PULL_PASSWORD
|
||||||
|
|
||||||
appwrite-worker-database:
|
appwrite-worker-database:
|
||||||
image: <?php echo $organization; ?>/<?php echo $image; ?>:<?php echo $version."\n"; ?>
|
image: <?php echo $organization; ?>/<?php echo $image; ?>:<?php echo $version."\n"; ?>
|
||||||
|
@ -324,13 +324,9 @@ services:
|
||||||
depends_on:
|
depends_on:
|
||||||
- redis
|
- redis
|
||||||
- mariadb
|
- mariadb
|
||||||
volumes:
|
- appwrite-executor
|
||||||
- /var/run/docker.sock:/var/run/docker.sock
|
|
||||||
- appwrite-functions:/storage/functions:rw
|
|
||||||
- /tmp:/tmp:rw
|
|
||||||
environment:
|
environment:
|
||||||
- _APP_ENV
|
- _APP_ENV
|
||||||
- _APP_OPENSSL_KEY_V1
|
|
||||||
- _APP_REDIS_HOST
|
- _APP_REDIS_HOST
|
||||||
- _APP_REDIS_PORT
|
- _APP_REDIS_PORT
|
||||||
- _APP_REDIS_USER
|
- _APP_REDIS_USER
|
||||||
|
@ -341,13 +337,7 @@ services:
|
||||||
- _APP_DB_USER
|
- _APP_DB_USER
|
||||||
- _APP_DB_PASS
|
- _APP_DB_PASS
|
||||||
- _APP_FUNCTIONS_TIMEOUT
|
- _APP_FUNCTIONS_TIMEOUT
|
||||||
- _APP_FUNCTIONS_BUILD_TIMEOUT
|
|
||||||
- _APP_FUNCTIONS_CONTAINERS
|
|
||||||
- _APP_FUNCTIONS_CPUS
|
|
||||||
- _APP_FUNCTIONS_MEMORY
|
|
||||||
- _APP_FUNCTIONS_MEMORY_SWAP
|
|
||||||
- _APP_EXECUTOR_SECRET
|
- _APP_EXECUTOR_SECRET
|
||||||
- _APP_FUNCTIONS_RUNTIMES
|
|
||||||
- _APP_USAGE_STATS
|
- _APP_USAGE_STATS
|
||||||
|
|
||||||
appwrite-executor:
|
appwrite-executor:
|
||||||
|
|
|
@ -1,10 +1,6 @@
|
||||||
<?php
|
<?php
|
||||||
|
|
||||||
use Appwrite\Event\Event;
|
|
||||||
use Appwrite\Messaging\Adapter\Realtime;
|
|
||||||
use Appwrite\Resque\Worker;
|
use Appwrite\Resque\Worker;
|
||||||
use Appwrite\Stats\Stats;
|
|
||||||
use Appwrite\Utopia\Response\Model\Execution;
|
|
||||||
use Cron\CronExpression;
|
use Cron\CronExpression;
|
||||||
use Swoole\Runtime;
|
use Swoole\Runtime;
|
||||||
use Utopia\App;
|
use Utopia\App;
|
||||||
|
@ -13,11 +9,8 @@ use Utopia\Config\Config;
|
||||||
use Utopia\Database\Database;
|
use Utopia\Database\Database;
|
||||||
use Utopia\Database\Document;
|
use Utopia\Database\Document;
|
||||||
use Utopia\Database\Validator\Authorization;
|
use Utopia\Database\Validator\Authorization;
|
||||||
use Utopia\Orchestration\Orchestration;
|
|
||||||
use Utopia\Orchestration\Adapter\DockerAPI;
|
use Utopia\Orchestration\Adapter\DockerAPI;
|
||||||
use Utopia\Orchestration\Container;
|
use Utopia\Orchestration\Orchestration;
|
||||||
use Utopia\Orchestration\Exception\Orchestration as OrchestrationException;
|
|
||||||
use Utopia\Orchestration\Exception\Timeout as TimeoutException;
|
|
||||||
|
|
||||||
require_once __DIR__.'/../init.php';
|
require_once __DIR__.'/../init.php';
|
||||||
|
|
||||||
|
@ -38,41 +31,6 @@ $warmupTime = $warmupEnd - $warmupStart;
|
||||||
|
|
||||||
Console::success('Finished warmup in ' . $warmupTime . ' seconds');
|
Console::success('Finished warmup in ' . $warmupTime . ' seconds');
|
||||||
|
|
||||||
/**
|
|
||||||
* List function servers
|
|
||||||
*/
|
|
||||||
$stdout = '';
|
|
||||||
$stderr = '';
|
|
||||||
|
|
||||||
$executionStart = \microtime(true);
|
|
||||||
|
|
||||||
$response = $orchestration->list(['label' => 'appwrite-type=function']);
|
|
||||||
/** @var Container[] $list */
|
|
||||||
$list = [];
|
|
||||||
|
|
||||||
foreach ($response as $value) {
|
|
||||||
$list[$value->getName()] = $value;
|
|
||||||
}
|
|
||||||
|
|
||||||
$executionEnd = \microtime(true);
|
|
||||||
|
|
||||||
Console::info(count($list) . ' functions listed in ' . ($executionEnd - $executionStart) . ' seconds');
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 1. Get event args - DONE
|
|
||||||
* 2. Unpackage code in the isolated container - DONE
|
|
||||||
* 3. Execute in container with timeout
|
|
||||||
* + messure execution time - DONE
|
|
||||||
* + pass env vars - DONE
|
|
||||||
* + pass one-time api key
|
|
||||||
* 4. Update execution status - DONE
|
|
||||||
* 5. Update execution stdout & stderr - DONE
|
|
||||||
* 6. Trigger audit log - DONE
|
|
||||||
* 7. Trigger usage log - DONE
|
|
||||||
*/
|
|
||||||
|
|
||||||
// TODO avoid scheduled execution if delay is bigger than X offest
|
|
||||||
|
|
||||||
class FunctionsV1 extends Worker
|
class FunctionsV1 extends Worker
|
||||||
{
|
{
|
||||||
public array $args = [];
|
public array $args = [];
|
||||||
|
@ -266,7 +224,7 @@ class FunctionsV1 extends Worker
|
||||||
public function execute(string $trigger, string $projectId, string $executionId, Database $database, Document $function, string $event = '', string $eventData = '', string $data = '', array $webhooks = [], string $userId = '', string $jwt = ''): void
|
public function execute(string $trigger, string $projectId, string $executionId, Database $database, Document $function, string $event = '', string $eventData = '', string $data = '', array $webhooks = [], string $userId = '', string $jwt = ''): void
|
||||||
{
|
{
|
||||||
$ch = \curl_init();
|
$ch = \curl_init();
|
||||||
\curl_setopt($ch, CURLOPT_URL, "http://appwrite-executor:8080/v1/execute");
|
\curl_setopt($ch, CURLOPT_URL, "http://appwrite-executor/v1/execute");
|
||||||
\curl_setopt($ch, CURLOPT_POST, true);
|
\curl_setopt($ch, CURLOPT_POST, true);
|
||||||
\curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode([
|
\curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode([
|
||||||
'trigger' => $trigger,
|
'trigger' => $trigger,
|
||||||
|
@ -299,67 +257,6 @@ class FunctionsV1 extends Worker
|
||||||
\curl_close($ch);
|
\curl_close($ch);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Cleanup any hanging containers above the allowed max containers.
|
|
||||||
*
|
|
||||||
* @return void
|
|
||||||
*/
|
|
||||||
public function cleanup(): void
|
|
||||||
{
|
|
||||||
/** @var Container[] $list */
|
|
||||||
global $list;
|
|
||||||
/** @var Orchestration $orchestration */
|
|
||||||
global $orchestration;
|
|
||||||
|
|
||||||
Console::success(count($list) . ' running containers counted');
|
|
||||||
|
|
||||||
$max = (int) App::getEnv('_APP_FUNCTIONS_CONTAINERS');
|
|
||||||
|
|
||||||
if (\count($list) > $max) {
|
|
||||||
Console::info('Starting containers cleanup');
|
|
||||||
|
|
||||||
\uasort($list, function (Container $item1, Container $item2) {
|
|
||||||
return (int)($item1->getLabels['appwrite-created'] ?? 0) <=> (int)($item2->getLabels['appwrite-created'] ?? 0);
|
|
||||||
});
|
|
||||||
|
|
||||||
while (\count($list) > $max) {
|
|
||||||
$first = \array_shift($list);
|
|
||||||
|
|
||||||
try {
|
|
||||||
$orchestration->remove($first->getName(), true);
|
|
||||||
Console::info('Removed container: ' . $first->getName());
|
|
||||||
} catch (Exception $e) {
|
|
||||||
Console::error('Failed to remove container: ' . $e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Filter ENV vars
|
|
||||||
*
|
|
||||||
* @param string $string
|
|
||||||
*
|
|
||||||
* @return string
|
|
||||||
*/
|
|
||||||
public function filterEnvKey(string $string): string
|
|
||||||
{
|
|
||||||
if (empty($this->allowed)) {
|
|
||||||
$this->allowed = array_fill_keys(\str_split('0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz_'), true);
|
|
||||||
}
|
|
||||||
|
|
||||||
$string = \str_split($string);
|
|
||||||
$output = '';
|
|
||||||
|
|
||||||
foreach ($string as $char) {
|
|
||||||
if (\array_key_exists($char, $this->allowed)) {
|
|
||||||
$output .= $char;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return $output;
|
|
||||||
}
|
|
||||||
|
|
||||||
public function shutdown(): void
|
public function shutdown(): void
|
||||||
{
|
{
|
||||||
}
|
}
|
||||||
|
|
|
@ -61,7 +61,6 @@ services:
|
||||||
- traefik.http.routers.appwrite_api_https.service=appwrite_api
|
- traefik.http.routers.appwrite_api_https.service=appwrite_api
|
||||||
- traefik.http.routers.appwrite_api_https.tls=true
|
- traefik.http.routers.appwrite_api_https.tls=true
|
||||||
volumes:
|
volumes:
|
||||||
- /var/run/docker.sock:/var/run/docker.sock
|
|
||||||
- appwrite-uploads:/storage/uploads:rw
|
- appwrite-uploads:/storage/uploads:rw
|
||||||
- appwrite-cache:/storage/cache:rw
|
- appwrite-cache:/storage/cache:rw
|
||||||
- appwrite-config:/storage/config:rw
|
- appwrite-config:/storage/config:rw
|
||||||
|
@ -326,17 +325,14 @@ services:
|
||||||
networks:
|
networks:
|
||||||
- appwrite
|
- appwrite
|
||||||
volumes:
|
volumes:
|
||||||
- /var/run/docker.sock:/var/run/docker.sock
|
|
||||||
- appwrite-functions:/storage/functions:rw
|
|
||||||
- /tmp:/tmp:rw
|
|
||||||
- ./app:/usr/src/code/app
|
- ./app:/usr/src/code/app
|
||||||
- ./src:/usr/src/code/src
|
- ./src:/usr/src/code/src
|
||||||
depends_on:
|
depends_on:
|
||||||
- redis
|
- redis
|
||||||
- mariadb
|
- mariadb
|
||||||
|
- appwrite-executor
|
||||||
environment:
|
environment:
|
||||||
- _APP_ENV
|
- _APP_ENV
|
||||||
- _APP_OPENSSL_KEY_V1
|
|
||||||
- _APP_REDIS_HOST
|
- _APP_REDIS_HOST
|
||||||
- _APP_REDIS_PORT
|
- _APP_REDIS_PORT
|
||||||
- _APP_REDIS_USER
|
- _APP_REDIS_USER
|
||||||
|
@ -347,12 +343,6 @@ services:
|
||||||
- _APP_DB_USER
|
- _APP_DB_USER
|
||||||
- _APP_DB_PASS
|
- _APP_DB_PASS
|
||||||
- _APP_FUNCTIONS_TIMEOUT
|
- _APP_FUNCTIONS_TIMEOUT
|
||||||
- _APP_FUNCTIONS_BUILD_TIMEOUT
|
|
||||||
- _APP_FUNCTIONS_CONTAINERS
|
|
||||||
- _APP_FUNCTIONS_RUNTIMES
|
|
||||||
- _APP_FUNCTIONS_CPUS
|
|
||||||
- _APP_FUNCTIONS_MEMORY
|
|
||||||
- _APP_FUNCTIONS_MEMORY_SWAP
|
|
||||||
- _APP_EXECUTOR_SECRET
|
- _APP_EXECUTOR_SECRET
|
||||||
- _APP_USAGE_STATS
|
- _APP_USAGE_STATS
|
||||||
- DOCKERHUB_PULL_USERNAME
|
- DOCKERHUB_PULL_USERNAME
|
||||||
|
|
Loading…
Reference in a new issue