Hiding port 8080
This commit is contained in:
parent
0219c92cb0
commit
029e1dc6f6
4 changed files with 285 additions and 287 deletions
|
@ -433,7 +433,7 @@ App::delete('/v1/functions/:functionId')
|
||||||
|
|
||||||
// Request executor to delete tag containers
|
// Request executor to delete tag containers
|
||||||
$ch = \curl_init();
|
$ch = \curl_init();
|
||||||
\curl_setopt($ch, CURLOPT_URL, "http://appwrite-executor:8080/v1/cleanup/function");
|
\curl_setopt($ch, CURLOPT_URL, "http://appwrite-executor/v1/cleanup/function");
|
||||||
\curl_setopt($ch, CURLOPT_POST, true);
|
\curl_setopt($ch, CURLOPT_POST, true);
|
||||||
\curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode([
|
\curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode([
|
||||||
'functionId' => $functionId
|
'functionId' => $functionId
|
||||||
|
@ -590,7 +590,7 @@ App::post('/v1/functions/:functionId/tags')
|
||||||
$function = $dbForProject->getDocument('functions', $functionId);
|
$function = $dbForProject->getDocument('functions', $functionId);
|
||||||
|
|
||||||
$ch = \curl_init();
|
$ch = \curl_init();
|
||||||
\curl_setopt($ch, CURLOPT_URL, "http://appwrite-executor:8080/v1/tag");
|
\curl_setopt($ch, CURLOPT_URL, "http://appwrite-executor/v1/tag");
|
||||||
\curl_setopt($ch, CURLOPT_POST, true);
|
\curl_setopt($ch, CURLOPT_POST, true);
|
||||||
\curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode([
|
\curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode([
|
||||||
'functionId' => $function->getId(),
|
'functionId' => $function->getId(),
|
||||||
|
@ -769,7 +769,7 @@ App::delete('/v1/functions/:functionId/tags/:tagId')
|
||||||
|
|
||||||
// Request executor to delete tag containers
|
// Request executor to delete tag containers
|
||||||
$ch = \curl_init();
|
$ch = \curl_init();
|
||||||
\curl_setopt($ch, CURLOPT_URL, "http://appwrite-executor:8080/v1/cleanup/tag");
|
\curl_setopt($ch, CURLOPT_URL, "http://appwrite-executor/v1/cleanup/tag");
|
||||||
\curl_setopt($ch, CURLOPT_POST, true);
|
\curl_setopt($ch, CURLOPT_POST, true);
|
||||||
\curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode([
|
\curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode([
|
||||||
'tagId' => $tagId
|
'tagId' => $tagId
|
||||||
|
@ -927,7 +927,7 @@ App::post('/v1/functions/:functionId/executions')
|
||||||
|
|
||||||
// Directly execute function.
|
// Directly execute function.
|
||||||
$ch = \curl_init();
|
$ch = \curl_init();
|
||||||
\curl_setopt($ch, CURLOPT_URL, "http://appwrite-executor:8080/v1/execute");
|
\curl_setopt($ch, CURLOPT_URL, "http://appwrite-executor/v1/execute");
|
||||||
\curl_setopt($ch, CURLOPT_POST, true);
|
\curl_setopt($ch, CURLOPT_POST, true);
|
||||||
\curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode([
|
\curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode([
|
||||||
'trigger' => 'http',
|
'trigger' => 'http',
|
||||||
|
@ -1162,7 +1162,7 @@ App::post('/v1/builds/:buildId')
|
||||||
|
|
||||||
// Retry build
|
// Retry build
|
||||||
$ch = \curl_init();
|
$ch = \curl_init();
|
||||||
\curl_setopt($ch, CURLOPT_URL, "http://appwrite-executor:8080/v1/build/{$buildId}");
|
\curl_setopt($ch, CURLOPT_URL, "http://appwrite-executor/v1/build/{$buildId}");
|
||||||
\curl_setopt($ch, CURLOPT_POST, true);
|
\curl_setopt($ch, CURLOPT_POST, true);
|
||||||
\curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
|
\curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
|
||||||
\curl_setopt($ch, CURLOPT_TIMEOUT, 900);
|
\curl_setopt($ch, CURLOPT_TIMEOUT, 900);
|
||||||
|
|
552
app/executor.php
552
app/executor.php
|
@ -1,40 +1,40 @@
|
||||||
<?php
|
<?php
|
||||||
require_once __DIR__ . '/../vendor/autoload.php';
|
require_once __DIR__ . '/../vendor/autoload.php';
|
||||||
|
|
||||||
use Utopia\App;
|
|
||||||
use Utopia\Swoole\Request;
|
|
||||||
use Utopia\Database\Document;
|
|
||||||
use Utopia\Database\Database;
|
|
||||||
use Utopia\Database\Validator\Authorization;
|
|
||||||
use Utopia\Database\Validator\UID;
|
|
||||||
use Appwrite\Event\Event;
|
use Appwrite\Event\Event;
|
||||||
use Appwrite\Utopia\Response\Model\Execution;
|
|
||||||
use Appwrite\Messaging\Adapter\Realtime;
|
use Appwrite\Messaging\Adapter\Realtime;
|
||||||
use Appwrite\Stats\Stats;
|
use Appwrite\Stats\Stats;
|
||||||
use Appwrite\Utopia\Response;
|
use Appwrite\Utopia\Response;
|
||||||
use Utopia\CLI\Console;
|
use Appwrite\Utopia\Response\Model\Execution;
|
||||||
use Swoole\Process;
|
use Cron\CronExpression;
|
||||||
|
use Swoole\ConnectionPool;
|
||||||
use Swoole\Coroutine as Co;
|
use Swoole\Coroutine as Co;
|
||||||
use Swoole\Http\Server;
|
|
||||||
use Swoole\Http\Request as SwooleRequest;
|
use Swoole\Http\Request as SwooleRequest;
|
||||||
use Swoole\Http\Response as SwooleResponse;
|
use Swoole\Http\Response as SwooleResponse;
|
||||||
use Utopia\Orchestration\Orchestration;
|
use Swoole\Http\Server;
|
||||||
use Utopia\Database\Adapter\MariaDB;
|
use Swoole\Process;
|
||||||
|
use Utopia\App;
|
||||||
|
use Utopia\CLI\Console;
|
||||||
use Utopia\Cache\Adapter\Redis as RedisCache;
|
use Utopia\Cache\Adapter\Redis as RedisCache;
|
||||||
|
use Utopia\Cache\Cache;
|
||||||
use Utopia\Config\Config;
|
use Utopia\Config\Config;
|
||||||
|
use Utopia\Database\Adapter\MariaDB;
|
||||||
|
use Utopia\Database\Database;
|
||||||
|
use Utopia\Database\Document;
|
||||||
|
use Utopia\Database\Query;
|
||||||
|
use Utopia\Database\Validator\Authorization;
|
||||||
|
use Utopia\Database\Validator\UID;
|
||||||
|
use Utopia\Logger\Log;
|
||||||
|
use Utopia\Orchestration\Adapter\DockerAPI;
|
||||||
|
use Utopia\Orchestration\Adapter\DockerCLI;
|
||||||
|
use Utopia\Orchestration\Orchestration;
|
||||||
|
use Utopia\Registry\Registry;
|
||||||
|
use Utopia\Storage\Device\Local;
|
||||||
|
use Utopia\Storage\Storage;
|
||||||
|
use Utopia\Swoole\Request;
|
||||||
use Utopia\Validator\ArrayList;
|
use Utopia\Validator\ArrayList;
|
||||||
use Utopia\Validator\JSON;
|
use Utopia\Validator\JSON;
|
||||||
use Utopia\Validator\Text;
|
use Utopia\Validator\Text;
|
||||||
use Cron\CronExpression;
|
|
||||||
use Swoole\ConnectionPool;
|
|
||||||
use Utopia\Storage\Device\Local;
|
|
||||||
use Utopia\Storage\Storage;
|
|
||||||
use Utopia\Cache\Cache;
|
|
||||||
use Utopia\Database\Query;
|
|
||||||
use Utopia\Orchestration\Adapter\DockerCLI;
|
|
||||||
use Utopia\Logger\Log;
|
|
||||||
use Utopia\Orchestration\Adapter\DockerAPI;
|
|
||||||
use Utopia\Registry\Registry;
|
|
||||||
|
|
||||||
require_once __DIR__ . '/init.php';
|
require_once __DIR__ . '/init.php';
|
||||||
|
|
||||||
|
@ -91,6 +91,7 @@ $orchestrationPool = new ConnectionPool(function () {
|
||||||
|
|
||||||
return $orchestration;
|
return $orchestration;
|
||||||
}, 6);
|
}, 6);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
$runtimes = Config::getParam('runtimes');
|
$runtimes = Config::getParam('runtimes');
|
||||||
|
|
||||||
|
@ -443,7 +444,7 @@ function execute(string $trigger, string $projectId, string $executionId, string
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
if (!$activeFunctions->exists($container)) { // Create contianer if not ready
|
if (!$activeFunctions->exists($container)) { // Create container if not ready
|
||||||
createRuntimeServer($functionId, $projectId, $tag->getId(), $database);
|
createRuntimeServer($functionId, $projectId, $tag->getId(), $database);
|
||||||
} else if ($activeFunctions->get($container)['status'] === 'Down') {
|
} else if ($activeFunctions->get($container)['status'] === 'Down') {
|
||||||
sleep(1);
|
sleep(1);
|
||||||
|
@ -651,6 +652,257 @@ function execute(string $trigger, string $projectId, string $executionId, string
|
||||||
];
|
];
|
||||||
};
|
};
|
||||||
|
|
||||||
|
function runBuildStage(string $buildId, string $projectID): Document
|
||||||
|
{
|
||||||
|
global $runtimes;
|
||||||
|
global $orchestrationPool;
|
||||||
|
global $register;
|
||||||
|
|
||||||
|
/** @var Orchestration $orchestration */
|
||||||
|
$orchestration = $orchestrationPool->get();
|
||||||
|
|
||||||
|
$buildStdout = '';
|
||||||
|
$buildStderr = '';
|
||||||
|
|
||||||
|
$db = $register->get('dbPool')->get();
|
||||||
|
$redis = $register->get('redisPool')->get();
|
||||||
|
$cache = new Cache(new RedisCache($redis));
|
||||||
|
|
||||||
|
$database = new Database(new MariaDB($db), $cache);
|
||||||
|
$database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite'));
|
||||||
|
$database->setNamespace('_project_' . $projectID);
|
||||||
|
|
||||||
|
// Check if build has already been run
|
||||||
|
$build = $database->getDocument('builds', $buildId);
|
||||||
|
|
||||||
|
try {
|
||||||
|
// If we already have a built package ready there is no need to rebuild.
|
||||||
|
if ($build->getAttribute('status') === 'ready' && \file_exists($build->getAttribute('outputPath'))) {
|
||||||
|
return $build;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Update Tag Status
|
||||||
|
$build->setAttribute('status', 'building');
|
||||||
|
|
||||||
|
$database->updateDocument('builds', $build->getId(), $build);
|
||||||
|
|
||||||
|
// Check if runtime is active
|
||||||
|
$runtime = $runtimes[$build->getAttribute('runtime', '')] ?? null;
|
||||||
|
|
||||||
|
if (\is_null($runtime)) {
|
||||||
|
throw new Exception('Runtime "' . $build->getAttribute('runtime', '') . '" is not supported');
|
||||||
|
}
|
||||||
|
|
||||||
|
// Grab Tag Files
|
||||||
|
$tagPath = $build->getAttribute('source', '');
|
||||||
|
$sourceType = $build->getAttribute('sourceType', '');
|
||||||
|
|
||||||
|
$device = Storage::getDevice('builds');
|
||||||
|
|
||||||
|
$tagPathTarget = '/tmp/project-' . $projectID . '/' . $build->getId() . '/code.tar.gz';
|
||||||
|
$tagPathTargetDir = \pathinfo($tagPathTarget, PATHINFO_DIRNAME);
|
||||||
|
|
||||||
|
$container = 'build-stage-' . $build->getId();
|
||||||
|
|
||||||
|
// Perform various checks
|
||||||
|
if (!\file_exists($tagPathTargetDir)) {
|
||||||
|
if (!\mkdir($tagPathTargetDir, 0777, true)) {
|
||||||
|
throw new Exception('Can\'t create directory ' . $tagPathTargetDir);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!\file_exists($tagPathTarget)) {
|
||||||
|
if (App::getEnv('_APP_STORAGE_DEVICE', Storage::DEVICE_LOCAL) === Storage::DEVICE_LOCAL) {
|
||||||
|
if (!\copy($tagPath, $tagPathTarget)) {
|
||||||
|
throw new Exception('Can\'t create temporary code file ' . $tagPathTarget);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
$buffer = $device->read($tagPath);
|
||||||
|
\file_put_contents($tagPathTarget, $buffer);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!$device->exists($tagPath)) {
|
||||||
|
throw new Exception('Code is not readable: ' . $build->getAttribute('source', ''));
|
||||||
|
}
|
||||||
|
|
||||||
|
$vars = $build->getAttribute('envVars', []);
|
||||||
|
|
||||||
|
// Start tracking time
|
||||||
|
$buildStart = \microtime(true);
|
||||||
|
$buildTime = \time();
|
||||||
|
|
||||||
|
$orchestration
|
||||||
|
->setCpus(App::getEnv('_APP_FUNCTIONS_CPUS', 0))
|
||||||
|
->setMemory(App::getEnv('_APP_FUNCTIONS_MEMORY', 256))
|
||||||
|
->setSwap(App::getEnv('_APP_FUNCTIONS_MEMORY_SWAP', 256));
|
||||||
|
|
||||||
|
foreach ($vars as &$value) {
|
||||||
|
$value = strval($value);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!\file_exists('/tmp/project-' . $projectID . '/' . $build->getId() . '/builtCode')) {
|
||||||
|
if (!\mkdir('/tmp/project-' . $projectID . '/' . $build->getId() . '/builtCode', 0777, true)) {
|
||||||
|
throw new Exception('Can\'t create directory /tmp/project-' . $projectID . '/' . $build->getId() . '/builtCode');
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
// Launch build container
|
||||||
|
$id = $orchestration->run(
|
||||||
|
image: $runtime['base'],
|
||||||
|
name: $container,
|
||||||
|
vars: $vars,
|
||||||
|
workdir: '/usr/code',
|
||||||
|
labels: [
|
||||||
|
'appwrite-type' => 'function',
|
||||||
|
'appwrite-created' => strval($buildTime),
|
||||||
|
'appwrite-runtime' => $build->getAttribute('runtime', ''),
|
||||||
|
'appwrite-project' => $projectID,
|
||||||
|
'appwrite-build' => $build->getId(),
|
||||||
|
],
|
||||||
|
command: [
|
||||||
|
'tail',
|
||||||
|
'-f',
|
||||||
|
'/dev/null'
|
||||||
|
],
|
||||||
|
hostname: $container,
|
||||||
|
mountFolder: $tagPathTargetDir,
|
||||||
|
volumes: [
|
||||||
|
'/tmp/project-' . $projectID . '/' . $build->getId() . '/builtCode' . ':/usr/builtCode:rw'
|
||||||
|
]
|
||||||
|
);
|
||||||
|
|
||||||
|
if (empty($id)) {
|
||||||
|
throw new Exception('Failed to start build container');
|
||||||
|
}
|
||||||
|
|
||||||
|
// Extract user code into build container
|
||||||
|
$untarStdout = '';
|
||||||
|
$untarStderr = '';
|
||||||
|
|
||||||
|
$untarSuccess = $orchestration->execute(
|
||||||
|
name: $container,
|
||||||
|
command: [
|
||||||
|
'sh',
|
||||||
|
'-c',
|
||||||
|
'mkdir -p /usr/code && cp /tmp/code.tar.gz /usr/workspace/code.tar.gz && cd /usr/workspace/ && tar -zxf /usr/workspace/code.tar.gz -C /usr/code && rm /usr/workspace/code.tar.gz'
|
||||||
|
],
|
||||||
|
stdout: $untarStdout,
|
||||||
|
stderr: $untarStderr,
|
||||||
|
timeout: 60
|
||||||
|
);
|
||||||
|
|
||||||
|
if (!$untarSuccess) {
|
||||||
|
throw new Exception('Failed to extract tar: ' . $untarStderr);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Build Code / Install Dependencies
|
||||||
|
$buildSuccess = $orchestration->execute(
|
||||||
|
name: $container,
|
||||||
|
command: ['sh', '-c', 'cd /usr/local/src && ./build.sh'],
|
||||||
|
stdout: $buildStdout,
|
||||||
|
stderr: $buildStderr,
|
||||||
|
timeout: App::getEnv('_APP_FUNCTIONS_BUILD_TIMEOUT', 900)
|
||||||
|
);
|
||||||
|
|
||||||
|
if (!$buildSuccess) {
|
||||||
|
throw new Exception('Failed to build dependencies: ' . $buildStderr);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Repackage Code and Save.
|
||||||
|
$compressStdout = '';
|
||||||
|
$compressStderr = '';
|
||||||
|
|
||||||
|
$builtCodePath = '/tmp/project-' . $projectID . '/' . $build->getId() . '/builtCode/code.tar.gz';
|
||||||
|
|
||||||
|
$compressSuccess = $orchestration->execute(
|
||||||
|
name: $container,
|
||||||
|
command: [
|
||||||
|
'tar', '-C', '/usr/code', '-czvf', '/usr/builtCode/code.tar.gz', './'
|
||||||
|
],
|
||||||
|
stdout: $compressStdout,
|
||||||
|
stderr: $compressStderr,
|
||||||
|
timeout: 60
|
||||||
|
);
|
||||||
|
|
||||||
|
if (!$compressSuccess) {
|
||||||
|
throw new Exception('Failed to compress built code: ' . $compressStderr);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Remove Container
|
||||||
|
$orchestration->remove($id, true);
|
||||||
|
|
||||||
|
// Check if the build was successful by checking if file exists
|
||||||
|
if (!\file_exists($builtCodePath)) {
|
||||||
|
throw new Exception('Something went wrong during the build process.');
|
||||||
|
}
|
||||||
|
|
||||||
|
// Upload new code
|
||||||
|
$device = Storage::getDevice('builds');
|
||||||
|
|
||||||
|
$path = $device->getPath(\uniqid() . '.' . \pathinfo('code.tar.gz', PATHINFO_EXTENSION));
|
||||||
|
|
||||||
|
if (!\file_exists(\dirname($path))) { // Checks if directory path to file exists
|
||||||
|
if (!@\mkdir(\dirname($path), 0777, true)) {
|
||||||
|
throw new Exception('Can\'t create directory: ' . \dirname($path));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (App::getEnv('_APP_STORAGE_DEVICE', Storage::DEVICE_LOCAL) === Storage::DEVICE_LOCAL) {
|
||||||
|
if (!$device->move($builtCodePath, $path)) {
|
||||||
|
throw new Exception('Failed to upload built code upload to storage', 500);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
if (!$device->upload($builtCodePath, $path)) {
|
||||||
|
throw new Exception('Failed to upload built code upload to storage', 500);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if ($buildStdout == '') {
|
||||||
|
$buildStdout = 'Build Successful!';
|
||||||
|
}
|
||||||
|
|
||||||
|
$build
|
||||||
|
->setAttribute('outputPath', $path)
|
||||||
|
->setAttribute('status', 'ready')
|
||||||
|
->setAttribute('stdout', \utf8_encode(\mb_substr($buildStdout, -4096)))
|
||||||
|
->setAttribute('stderr', \utf8_encode(\mb_substr($buildStderr, -4096)))
|
||||||
|
->setAttribute('buildTime', $buildTime);
|
||||||
|
|
||||||
|
// Update build with built code attribute
|
||||||
|
$build = $database->updateDocument('builds', $buildId, $build);
|
||||||
|
|
||||||
|
$buildEnd = \microtime(true);
|
||||||
|
|
||||||
|
Console::info('Build Stage Ran in ' . ($buildEnd - $buildStart) . ' seconds');
|
||||||
|
} catch (Exception $e) {
|
||||||
|
$build
|
||||||
|
->setAttribute('status', 'failed')
|
||||||
|
->setAttribute('stdout', \utf8_encode(\mb_substr($buildStdout, -4096)))
|
||||||
|
->setAttribute('stderr', \utf8_encode(\mb_substr($e->getMessage(), -4096)));
|
||||||
|
|
||||||
|
$build = $database->updateDocument('builds', $buildId, $build);
|
||||||
|
|
||||||
|
// also remove the container if it exists
|
||||||
|
if (isset($id)) {
|
||||||
|
$orchestration->remove($id, true);
|
||||||
|
}
|
||||||
|
$orchestrationPool->put(null);
|
||||||
|
|
||||||
|
$register->get('dbPool')->put($db);
|
||||||
|
$register->get('redisPool')->put($redis);
|
||||||
|
|
||||||
|
throw new Exception('Build failed: ' . $e->getMessage());
|
||||||
|
} finally {
|
||||||
|
$orchestrationPool->put($orchestration);
|
||||||
|
|
||||||
|
$register->get('dbPool')->put($db);
|
||||||
|
$register->get('redisPool')->put($redis);
|
||||||
|
}
|
||||||
|
|
||||||
|
return $build;
|
||||||
|
}
|
||||||
|
|
||||||
App::post('/v1/execute') // Define Route
|
App::post('/v1/execute') // Define Route
|
||||||
->desc('Execute a function')
|
->desc('Execute a function')
|
||||||
->param('trigger', '', new Text(1024))
|
->param('trigger', '', new Text(1024))
|
||||||
|
@ -682,7 +934,6 @@ App::post('/v1/execute') // Define Route
|
||||||
}
|
}
|
||||||
);
|
);
|
||||||
|
|
||||||
|
|
||||||
// Cleanup Endpoints used internally by appwrite when a function or tag gets deleted to also clean up their containers
|
// Cleanup Endpoints used internally by appwrite when a function or tag gets deleted to also clean up their containers
|
||||||
App::post('/v1/cleanup/function')
|
App::post('/v1/cleanup/function')
|
||||||
->param('functionId', '', new UID())
|
->param('functionId', '', new UID())
|
||||||
|
@ -961,260 +1212,9 @@ App::post('/v1/build/:buildId') // Start a Build
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
function runBuildStage(string $buildId, string $projectID): Document
|
|
||||||
{
|
|
||||||
global $runtimes;
|
|
||||||
global $orchestrationPool;
|
|
||||||
global $register;
|
|
||||||
|
|
||||||
/** @var Orchestration $orchestration */
|
|
||||||
$orchestration = $orchestrationPool->get();
|
|
||||||
|
|
||||||
$buildStdout = '';
|
|
||||||
$buildStderr = '';
|
|
||||||
|
|
||||||
$db = $register->get('dbPool')->get();
|
|
||||||
$redis = $register->get('redisPool')->get();
|
|
||||||
$cache = new Cache(new RedisCache($redis));
|
|
||||||
|
|
||||||
$database = new Database(new MariaDB($db), $cache);
|
|
||||||
$database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite'));
|
|
||||||
$database->setNamespace('_project_' . $projectID);
|
|
||||||
|
|
||||||
// Check if build has already been run
|
|
||||||
$build = $database->getDocument('builds', $buildId);
|
|
||||||
|
|
||||||
try {
|
|
||||||
// If we already have a built package ready there is no need to rebuild.
|
|
||||||
if ($build->getAttribute('status') === 'ready' && \file_exists($build->getAttribute('outputPath'))) {
|
|
||||||
return $build;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Update Tag Status
|
|
||||||
$build->setAttribute('status', 'building');
|
|
||||||
|
|
||||||
$database->updateDocument('builds', $build->getId(), $build);
|
|
||||||
|
|
||||||
// Check if runtime is active
|
|
||||||
$runtime = $runtimes[$build->getAttribute('runtime', '')] ?? null;
|
|
||||||
|
|
||||||
if (\is_null($runtime)) {
|
|
||||||
throw new Exception('Runtime "' . $build->getAttribute('runtime', '') . '" is not supported');
|
|
||||||
}
|
|
||||||
|
|
||||||
// Grab Tag Files
|
|
||||||
$tagPath = $build->getAttribute('source', '');
|
|
||||||
$sourceType = $build->getAttribute('sourceType', '');
|
|
||||||
|
|
||||||
$device = Storage::getDevice('builds');
|
|
||||||
|
|
||||||
$tagPathTarget = '/tmp/project-' . $projectID . '/' . $build->getId() . '/code.tar.gz';
|
|
||||||
$tagPathTargetDir = \pathinfo($tagPathTarget, PATHINFO_DIRNAME);
|
|
||||||
|
|
||||||
$container = 'build-stage-' . $build->getId();
|
|
||||||
|
|
||||||
// Perform various checks
|
|
||||||
if (!\file_exists($tagPathTargetDir)) {
|
|
||||||
if (!\mkdir($tagPathTargetDir, 0777, true)) {
|
|
||||||
throw new Exception('Can\'t create directory ' . $tagPathTargetDir);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!\file_exists($tagPathTarget)) {
|
|
||||||
if (App::getEnv('_APP_STORAGE_DEVICE', Storage::DEVICE_LOCAL) === Storage::DEVICE_LOCAL) {
|
|
||||||
if (!\copy($tagPath, $tagPathTarget)) {
|
|
||||||
throw new Exception('Can\'t create temporary code file ' . $tagPathTarget);
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
$buffer = $device->read($tagPath);
|
|
||||||
\file_put_contents($tagPathTarget, $buffer);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!$device->exists($tagPath)) {
|
|
||||||
throw new Exception('Code is not readable: ' . $build->getAttribute('source', ''));
|
|
||||||
}
|
|
||||||
|
|
||||||
$vars = $build->getAttribute('envVars', []);
|
|
||||||
|
|
||||||
// Start tracking time
|
|
||||||
$buildStart = \microtime(true);
|
|
||||||
$buildTime = \time();
|
|
||||||
|
|
||||||
$orchestration
|
|
||||||
->setCpus(App::getEnv('_APP_FUNCTIONS_CPUS', 0))
|
|
||||||
->setMemory(App::getEnv('_APP_FUNCTIONS_MEMORY', 256))
|
|
||||||
->setSwap(App::getEnv('_APP_FUNCTIONS_MEMORY_SWAP', 256));
|
|
||||||
|
|
||||||
foreach ($vars as &$value) {
|
|
||||||
$value = strval($value);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!\file_exists('/tmp/project-' . $projectID . '/' . $build->getId() . '/builtCode')) {
|
|
||||||
if (!\mkdir('/tmp/project-' . $projectID . '/' . $build->getId() . '/builtCode', 0777, true)) {
|
|
||||||
throw new Exception('Can\'t create directory /tmp/project-' . $projectID . '/' . $build->getId() . '/builtCode');
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
// Launch build container
|
|
||||||
$id = $orchestration->run(
|
|
||||||
image: $runtime['base'],
|
|
||||||
name: $container,
|
|
||||||
vars: $vars,
|
|
||||||
workdir: '/usr/code',
|
|
||||||
labels: [
|
|
||||||
'appwrite-type' => 'function',
|
|
||||||
'appwrite-created' => strval($buildTime),
|
|
||||||
'appwrite-runtime' => $build->getAttribute('runtime', ''),
|
|
||||||
'appwrite-project' => $projectID,
|
|
||||||
'appwrite-build' => $build->getId(),
|
|
||||||
],
|
|
||||||
command: [
|
|
||||||
'tail',
|
|
||||||
'-f',
|
|
||||||
'/dev/null'
|
|
||||||
],
|
|
||||||
hostname: $container,
|
|
||||||
mountFolder: $tagPathTargetDir,
|
|
||||||
volumes: [
|
|
||||||
'/tmp/project-' . $projectID . '/' . $build->getId() . '/builtCode' . ':/usr/builtCode:rw'
|
|
||||||
]
|
|
||||||
);
|
|
||||||
|
|
||||||
if (empty($id)) {
|
|
||||||
throw new Exception('Failed to start build container');
|
|
||||||
}
|
|
||||||
|
|
||||||
// Extract user code into build container
|
|
||||||
$untarStdout = '';
|
|
||||||
$untarStderr = '';
|
|
||||||
|
|
||||||
$untarSuccess = $orchestration->execute(
|
|
||||||
name: $container,
|
|
||||||
command: [
|
|
||||||
'sh',
|
|
||||||
'-c',
|
|
||||||
'mkdir -p /usr/code && cp /tmp/code.tar.gz /usr/workspace/code.tar.gz && cd /usr/workspace/ && tar -zxf /usr/workspace/code.tar.gz -C /usr/code && rm /usr/workspace/code.tar.gz'
|
|
||||||
],
|
|
||||||
stdout: $untarStdout,
|
|
||||||
stderr: $untarStderr,
|
|
||||||
timeout: 60
|
|
||||||
);
|
|
||||||
|
|
||||||
if (!$untarSuccess) {
|
|
||||||
throw new Exception('Failed to extract tar: ' . $untarStderr);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Build Code / Install Dependencies
|
|
||||||
$buildSuccess = $orchestration->execute(
|
|
||||||
name: $container,
|
|
||||||
command: ['sh', '-c', 'cd /usr/local/src && ./build.sh'],
|
|
||||||
stdout: $buildStdout,
|
|
||||||
stderr: $buildStderr,
|
|
||||||
timeout: App::getEnv('_APP_FUNCTIONS_BUILD_TIMEOUT', 900)
|
|
||||||
);
|
|
||||||
|
|
||||||
if (!$buildSuccess) {
|
|
||||||
throw new Exception('Failed to build dependencies: ' . $buildStderr);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Repackage Code and Save.
|
|
||||||
$compressStdout = '';
|
|
||||||
$compressStderr = '';
|
|
||||||
|
|
||||||
$builtCodePath = '/tmp/project-' . $projectID . '/' . $build->getId() . '/builtCode/code.tar.gz';
|
|
||||||
|
|
||||||
$compressSuccess = $orchestration->execute(
|
|
||||||
name: $container,
|
|
||||||
command: [
|
|
||||||
'tar', '-C', '/usr/code', '-czvf', '/usr/builtCode/code.tar.gz', './'
|
|
||||||
],
|
|
||||||
stdout: $compressStdout,
|
|
||||||
stderr: $compressStderr,
|
|
||||||
timeout: 60
|
|
||||||
);
|
|
||||||
|
|
||||||
if (!$compressSuccess) {
|
|
||||||
throw new Exception('Failed to compress built code: ' . $compressStderr);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Remove Container
|
|
||||||
$orchestration->remove($id, true);
|
|
||||||
|
|
||||||
// Check if the build was successful by checking if file exists
|
|
||||||
if (!\file_exists($builtCodePath)) {
|
|
||||||
throw new Exception('Something went wrong during the build process.');
|
|
||||||
}
|
|
||||||
|
|
||||||
// Upload new code
|
|
||||||
$device = Storage::getDevice('builds');
|
|
||||||
|
|
||||||
$path = $device->getPath(\uniqid() . '.' . \pathinfo('code.tar.gz', PATHINFO_EXTENSION));
|
|
||||||
|
|
||||||
if (!\file_exists(\dirname($path))) { // Checks if directory path to file exists
|
|
||||||
if (!@\mkdir(\dirname($path), 0777, true)) {
|
|
||||||
throw new Exception('Can\'t create directory: ' . \dirname($path));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (App::getEnv('_APP_STORAGE_DEVICE', Storage::DEVICE_LOCAL) === Storage::DEVICE_LOCAL) {
|
|
||||||
if (!$device->move($builtCodePath, $path)) {
|
|
||||||
throw new Exception('Failed to upload built code upload to storage', 500);
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
if (!$device->upload($builtCodePath, $path)) {
|
|
||||||
throw new Exception('Failed to upload built code upload to storage', 500);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if ($buildStdout == '') {
|
|
||||||
$buildStdout = 'Build Successful!';
|
|
||||||
}
|
|
||||||
|
|
||||||
$build
|
|
||||||
->setAttribute('outputPath', $path)
|
|
||||||
->setAttribute('status', 'ready')
|
|
||||||
->setAttribute('stdout', \utf8_encode(\mb_substr($buildStdout, -4096)))
|
|
||||||
->setAttribute('stderr', \utf8_encode(\mb_substr($buildStderr, -4096)))
|
|
||||||
->setAttribute('buildTime', $buildTime);
|
|
||||||
|
|
||||||
// Update build with built code attribute
|
|
||||||
$build = $database->updateDocument('builds', $buildId, $build);
|
|
||||||
|
|
||||||
$buildEnd = \microtime(true);
|
|
||||||
|
|
||||||
Console::info('Build Stage Ran in ' . ($buildEnd - $buildStart) . ' seconds');
|
|
||||||
} catch (Exception $e) {
|
|
||||||
$build
|
|
||||||
->setAttribute('status', 'failed')
|
|
||||||
->setAttribute('stdout', \utf8_encode(\mb_substr($buildStdout, -4096)))
|
|
||||||
->setAttribute('stderr', \utf8_encode(\mb_substr($e->getMessage(), -4096)));
|
|
||||||
|
|
||||||
$build = $database->updateDocument('builds', $buildId, $build);
|
|
||||||
|
|
||||||
// also remove the container if it exists
|
|
||||||
if (isset($id)) {
|
|
||||||
$orchestration->remove($id, true);
|
|
||||||
}
|
|
||||||
$orchestrationPool->put(null);
|
|
||||||
|
|
||||||
$register->get('dbPool')->put($db);
|
|
||||||
$register->get('redisPool')->put($redis);
|
|
||||||
|
|
||||||
throw new Exception('Build failed: ' . $e->getMessage());
|
|
||||||
} finally {
|
|
||||||
$orchestrationPool->put($orchestration);
|
|
||||||
|
|
||||||
$register->get('dbPool')->put($db);
|
|
||||||
$register->get('redisPool')->put($redis);
|
|
||||||
}
|
|
||||||
|
|
||||||
return $build;
|
|
||||||
}
|
|
||||||
|
|
||||||
App::setMode(App::MODE_TYPE_PRODUCTION); // Define Mode
|
App::setMode(App::MODE_TYPE_PRODUCTION); // Define Mode
|
||||||
|
|
||||||
$http = new Server("0.0.0.0", 8080);
|
$http = new Server("0.0.0.0", 80);
|
||||||
|
|
||||||
function handleShutdown()
|
function handleShutdown()
|
||||||
{
|
{
|
||||||
|
|
|
@ -9,8 +9,8 @@ use Utopia\Config\Config;
|
||||||
use Utopia\Database\Database;
|
use Utopia\Database\Database;
|
||||||
use Utopia\Database\Document;
|
use Utopia\Database\Document;
|
||||||
use Utopia\Database\Validator\Authorization;
|
use Utopia\Database\Validator\Authorization;
|
||||||
use Utopia\Orchestration\Orchestration;
|
|
||||||
use Utopia\Orchestration\Adapter\DockerAPI;
|
use Utopia\Orchestration\Adapter\DockerAPI;
|
||||||
|
use Utopia\Orchestration\Orchestration;
|
||||||
|
|
||||||
require_once __DIR__.'/../init.php';
|
require_once __DIR__.'/../init.php';
|
||||||
|
|
||||||
|
@ -224,7 +224,7 @@ class FunctionsV1 extends Worker
|
||||||
public function execute(string $trigger, string $projectId, string $executionId, Database $database, Document $function, string $event = '', string $eventData = '', string $data = '', array $webhooks = [], string $userId = '', string $jwt = ''): void
|
public function execute(string $trigger, string $projectId, string $executionId, Database $database, Document $function, string $event = '', string $eventData = '', string $data = '', array $webhooks = [], string $userId = '', string $jwt = ''): void
|
||||||
{
|
{
|
||||||
$ch = \curl_init();
|
$ch = \curl_init();
|
||||||
\curl_setopt($ch, CURLOPT_URL, "http://appwrite-executor:8080/v1/execute");
|
\curl_setopt($ch, CURLOPT_URL, "http://appwrite-executor/v1/execute");
|
||||||
\curl_setopt($ch, CURLOPT_POST, true);
|
\curl_setopt($ch, CURLOPT_POST, true);
|
||||||
\curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode([
|
\curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode([
|
||||||
'trigger' => $trigger,
|
'trigger' => $trigger,
|
||||||
|
|
|
@ -367,8 +367,6 @@ services:
|
||||||
- /usr/src/code/app/executor.php
|
- /usr/src/code/app/executor.php
|
||||||
- -dopcache.preload=opcache.preload=/usr/src/code/app/preload.php
|
- -dopcache.preload=opcache.preload=/usr/src/code/app/preload.php
|
||||||
stop_signal: SIGINT
|
stop_signal: SIGINT
|
||||||
ports:
|
|
||||||
- "8080:8080"
|
|
||||||
build:
|
build:
|
||||||
context: .
|
context: .
|
||||||
args:
|
args:
|
||||||
|
@ -411,10 +409,10 @@ services:
|
||||||
- _APP_USAGE_STATS
|
- _APP_USAGE_STATS
|
||||||
- _APP_STATSD_HOST
|
- _APP_STATSD_HOST
|
||||||
- _APP_STATSD_PORT
|
- _APP_STATSD_PORT
|
||||||
- DOCKERHUB_PULL_USERNAME
|
|
||||||
- DOCKERHUB_PULL_PASSWORD
|
|
||||||
- _APP_LOGGING_PROVIDER
|
- _APP_LOGGING_PROVIDER
|
||||||
- _APP_LOGGING_CONFIG
|
- _APP_LOGGING_CONFIG
|
||||||
|
- DOCKERHUB_PULL_USERNAME
|
||||||
|
- DOCKERHUB_PULL_PASSWORD
|
||||||
|
|
||||||
appwrite-worker-mails:
|
appwrite-worker-mails:
|
||||||
entrypoint: worker-mails
|
entrypoint: worker-mails
|
||||||
|
|
Loading…
Reference in a new issue