Add build step and more function cleanup
This commit is contained in:
parent
66d8b0599c
commit
da8b4bb9f9
|
@ -1622,6 +1622,24 @@ $collections = [
|
|||
'default' => '',
|
||||
'required' => false,
|
||||
'array' => false,
|
||||
],
|
||||
[
|
||||
'$collection' => Database::SYSTEM_COLLECTION_RULES,
|
||||
'label' => 'Build Status',
|
||||
'key' => 'status',
|
||||
'type' => Database::SYSTEM_VAR_TYPE_TEXT,
|
||||
'default' => '',
|
||||
'required' => false,
|
||||
'array' => false,
|
||||
],
|
||||
[
|
||||
'$collection' => Database::SYSTEM_COLLECTION_RULES,
|
||||
'label' => 'Build Path',
|
||||
'key' => 'builtPath',
|
||||
'type' => Database::SYSTEM_VAR_TYPE_TEXT,
|
||||
'default' => '',
|
||||
'required' => false,
|
||||
'array' => false,
|
||||
]
|
||||
],
|
||||
],
|
||||
|
|
|
@ -403,15 +403,47 @@ App::delete('/v1/functions/:functionId')
|
|||
->label('sdk.response.model', Response::MODEL_NONE)
|
||||
->param('functionId', '', new UID(), 'Function unique ID.')
|
||||
->inject('response')
|
||||
->inject('project')
|
||||
->inject('projectDB')
|
||||
->inject('deletes')
|
||||
->action(function ($functionId, $response, $projectDB, $deletes) {
|
||||
->action(function ($functionId, $response, $project, $projectDB, $deletes) {
|
||||
/** @var Appwrite\Utopia\Response $response */
|
||||
/** @var Appwrite\Database\Database $projectDB */
|
||||
/** @var Appwrite\Event\Event $deletes */
|
||||
|
||||
$function = $projectDB->getDocument($functionId);
|
||||
|
||||
// Request executor to delete tag containers
|
||||
$ch = \curl_init();
|
||||
\curl_setopt($ch, CURLOPT_URL, "http://appwrite-executor:8080/v1/cleanup/function");
|
||||
\curl_setopt($ch, CURLOPT_POST, true);
|
||||
\curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode([
|
||||
'functionId' => $functionId
|
||||
]));
|
||||
\curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
|
||||
\curl_setopt($ch, CURLOPT_TIMEOUT, 900);
|
||||
\curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 10);
|
||||
\curl_setopt($ch, CURLOPT_HTTPHEADER, [
|
||||
'Content-Type: application/json',
|
||||
'X-Appwrite-Project: '.$project->getId(),
|
||||
]);
|
||||
|
||||
$executorResponse = \curl_exec($ch);
|
||||
|
||||
$error = \curl_error($ch);
|
||||
|
||||
if (!empty($error)) {
|
||||
throw new Exception('Curl error: ' . $error, 500);
|
||||
}
|
||||
|
||||
// Check status code
|
||||
$statusCode = \curl_getinfo($ch, CURLINFO_HTTP_CODE);
|
||||
if (200 !== $statusCode) {
|
||||
throw new Exception('Executor error: ' . $executorResponse, $statusCode);
|
||||
}
|
||||
|
||||
\curl_close($ch);
|
||||
|
||||
if (empty($function->getId()) || Database::SYSTEM_COLLECTION_FUNCTIONS != $function->getCollection()) {
|
||||
throw new Exception('Function not found', 404);
|
||||
}
|
||||
|
@ -506,7 +538,9 @@ App::post('/v1/functions/:functionId/tags')
|
|||
'dateCreated' => time(),
|
||||
'entrypoint' => $entrypoint,
|
||||
'path' => $path,
|
||||
'size' => $size
|
||||
'size' => $size,
|
||||
'status' => 'pending',
|
||||
'builtPath' => ''
|
||||
]);
|
||||
|
||||
if (false === $tag) {
|
||||
|
@ -620,9 +654,10 @@ App::delete('/v1/functions/:functionId/tags/:tagId')
|
|||
->param('functionId', '', new UID(), 'Function unique ID.')
|
||||
->param('tagId', '', new UID(), 'Tag unique ID.')
|
||||
->inject('response')
|
||||
->inject('project')
|
||||
->inject('projectDB')
|
||||
->inject('usage')
|
||||
->action(function ($functionId, $tagId, $response, $projectDB, $usage) {
|
||||
->action(function ($functionId, $tagId, $response, $project, $projectDB, $usage) {
|
||||
/** @var Appwrite\Utopia\Response $response */
|
||||
/** @var Appwrite\Database\Database $projectDB */
|
||||
/** @var Appwrite\Event\Event $usage */
|
||||
|
@ -643,6 +678,37 @@ App::delete('/v1/functions/:functionId/tags/:tagId')
|
|||
throw new Exception('Tag not found', 404);
|
||||
}
|
||||
|
||||
// Request executor to delete tag containers
|
||||
$ch = \curl_init();
|
||||
\curl_setopt($ch, CURLOPT_URL, "http://appwrite-executor:8080/v1/cleanup/tag");
|
||||
\curl_setopt($ch, CURLOPT_POST, true);
|
||||
\curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode([
|
||||
'tagId' => $tagId
|
||||
]));
|
||||
\curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
|
||||
\curl_setopt($ch, CURLOPT_TIMEOUT, 900);
|
||||
\curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 10);
|
||||
\curl_setopt($ch, CURLOPT_HTTPHEADER, [
|
||||
'Content-Type: application/json',
|
||||
'X-Appwrite-Project: '.$project->getId(),
|
||||
]);
|
||||
|
||||
$executorResponse = \curl_exec($ch);
|
||||
|
||||
$error = \curl_error($ch);
|
||||
|
||||
if (!empty($error)) {
|
||||
throw new Exception('Curl error: ' . $error, 500);
|
||||
}
|
||||
|
||||
// Check status code
|
||||
$statusCode = \curl_getinfo($ch, CURLINFO_HTTP_CODE);
|
||||
if (200 !== $statusCode) {
|
||||
throw new Exception('Executor error: ' . $executorResponse, $statusCode);
|
||||
}
|
||||
|
||||
\curl_close($ch);
|
||||
|
||||
$device = Storage::getDevice('functions');
|
||||
|
||||
if ($device->delete($tag->getAttribute('path', ''))) {
|
||||
|
|
487
app/executor.php
487
app/executor.php
|
@ -26,6 +26,12 @@ use Utopia\Validator\ArrayList;
|
|||
use Utopia\Validator\JSON;
|
||||
use Utopia\Validator\Text;
|
||||
use Cron\CronExpression;
|
||||
use Utopia\Storage\Device\Local;
|
||||
use Utopia\Storage\Storage;
|
||||
use Utopia\Storage\Validator\FileExt;
|
||||
use Utopia\Storage\Validator\FileSize;
|
||||
use Utopia\Storage\Validator\Upload;
|
||||
use Swoole\Coroutine as Co;
|
||||
|
||||
require_once __DIR__ . '/workers.php';
|
||||
|
||||
|
@ -39,12 +45,12 @@ $runtimes = Config::getParam('runtimes');
|
|||
Swoole\Runtime::enableCoroutine(true, SWOOLE_HOOK_ALL ^ SWOOLE_HOOK_CURL);
|
||||
|
||||
// Warmup: make sure images are ready to run fast 🚀
|
||||
Co\run(function() use ($runtimes, $orchestration) {
|
||||
foreach($runtimes as $runtime) {
|
||||
go(function() use ($runtime, $orchestration) {
|
||||
Console::info('Warming up '.$runtime['name'].' '.$runtime['version'].' environment...');
|
||||
|
||||
$response = $orchestration->pull($runtime['image']);
|
||||
Co\run(function () use ($runtimes, $orchestration) {
|
||||
foreach ($runtimes as $runtime) {
|
||||
go(function () use ($runtime, $orchestration) {
|
||||
Console::info('Warming up ' . $runtime['name'] . ' ' . $runtime['version'] . ' environment...');
|
||||
|
||||
$response = $orchestration->pull($runtime['image']);
|
||||
|
||||
if ($response) {
|
||||
Console::success("Successfully Warmed up {$runtime['name']} {$runtime['version']}!");
|
||||
|
@ -70,7 +76,7 @@ foreach ($response as $value) {
|
|||
|
||||
$executionEnd = \microtime(true);
|
||||
|
||||
Console::info(count($activeFunctions).' functions listed in ' . ($executionEnd - $executionStart) . ' seconds');
|
||||
Console::info(count($activeFunctions) . ' functions listed in ' . ($executionEnd - $executionStart) . ' seconds');
|
||||
|
||||
App::post('/v1/execute') // Define Route
|
||||
->inject('request')
|
||||
|
@ -87,19 +93,127 @@ App::post('/v1/execute') // Define Route
|
|||
->inject('response')
|
||||
->action(
|
||||
function ($trigger, $projectId, $executionId, $functionId, $event, $eventData, $data, $webhooks, $userId, $jwt, $request, $response) {
|
||||
global $register;
|
||||
|
||||
$db = $register->get('dbPool')->get();
|
||||
$cache = $register->get('redisPool')->get();
|
||||
|
||||
// Create new Database Instance
|
||||
$database = new Database();
|
||||
$database->setAdapter(new RedisAdapter(new MySQLAdapter($db, $cache), $cache));
|
||||
$database->setNamespace('app_' . $projectId);
|
||||
$database->setMocks(Config::getParam('collections', []));
|
||||
|
||||
try {
|
||||
$data = execute($trigger, $projectId, $executionId, $functionId, $event, $eventData, $data, $webhooks, $userId, $jwt);
|
||||
return $response->json($data);
|
||||
$data = execute($trigger, $projectId, $executionId, $functionId, $database, $event, $eventData, $data, $webhooks, $userId, $jwt);
|
||||
$response->json($data);
|
||||
} catch (Exception $e) {
|
||||
return $response
|
||||
$response
|
||||
->addHeader('Cache-Control', 'no-cache, no-store, must-revalidate')
|
||||
->addHeader('Expires', '0')
|
||||
->addHeader('Pragma', 'no-cache')
|
||||
->json(['error' => $e->getMessage()]);
|
||||
} finally {
|
||||
$register->get('dbPool')->put($db);
|
||||
$register->get('redisPool')->put($cache);
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
|
||||
// Cleanup Endpoints used internally by appwrite when a function or tag gets deleted to also clean up their containers
|
||||
App::post('/v1/cleanup/function')
|
||||
->param('functionId', '', new UID())
|
||||
->inject('response')
|
||||
->inject('projectDB')
|
||||
->inject('projectID')
|
||||
->action(function ($functionId, $response, $projectDB, $projectID) {
|
||||
/** @var string $functionId */
|
||||
/** @var Appwrite\Utopia\Response $response */
|
||||
/** @var Appwrite\Database\Database $projectDB */
|
||||
/** @var string $projectID */
|
||||
|
||||
global $orchestration;
|
||||
|
||||
try {
|
||||
Authorization::disable();
|
||||
$function = $projectDB->getDocument($functionId);
|
||||
Authorization::reset();
|
||||
|
||||
if (\is_null($function->getId()) || Database::SYSTEM_COLLECTION_FUNCTIONS != $function->getCollection()) {
|
||||
throw new Exception('Function not found', 404);
|
||||
}
|
||||
|
||||
Authorization::disable();
|
||||
$results = $projectDB->getCollection([
|
||||
'limit' => 999,
|
||||
'offset' => 0,
|
||||
'orderType' => 'ASC',
|
||||
'filters' => [
|
||||
'$collection='.Database::SYSTEM_COLLECTION_TAGS,
|
||||
'functionId='.$functionId,
|
||||
],
|
||||
]);
|
||||
Authorization::reset();
|
||||
|
||||
// If amount is 0 then we simply return true
|
||||
if (count($results) === 0) {
|
||||
return $response->json(['success' => true]);
|
||||
}
|
||||
|
||||
// Delete the containers of all tags
|
||||
foreach ($results as $tag) {
|
||||
try {
|
||||
$orchestration->remove('appwrite-function-'.$tag['$id'], true);
|
||||
Console::info('Removed container for tag ' . $tag['$id']);
|
||||
} catch (Exception $e) {
|
||||
// Do nothing, we don't care that much if it fails
|
||||
}
|
||||
}
|
||||
|
||||
return $response->json(['success' => true]);
|
||||
} catch (Exception $e) {
|
||||
Console::error($e->getMessage());
|
||||
return $response->json(['error' => $e->getMessage()]);
|
||||
}
|
||||
});
|
||||
|
||||
App::post('/v1/cleanup/tag')
|
||||
->param('tagId', '', new UID(), 'Tag unique ID.')
|
||||
->inject('response')
|
||||
->inject('projectDB')
|
||||
->inject('projectID')
|
||||
->action(function ($tagId, $response, $projectDB, $projectID) {
|
||||
/** @var string $tagId */
|
||||
/** @var Appwrite\Utopia\Response $response */
|
||||
/** @var Appwrite\Database\Database $projectDB */
|
||||
/** @var string $projectID */
|
||||
|
||||
global $orchestration;
|
||||
|
||||
try {
|
||||
Authorization::disable();
|
||||
$tag = $projectDB->getDocument($tagId);
|
||||
Authorization::reset();
|
||||
|
||||
if (\is_null($tag->getId()) || Database::SYSTEM_COLLECTION_TAGS != $tag->getCollection()) {
|
||||
throw new Exception('Tag not found', 404);
|
||||
}
|
||||
|
||||
try {
|
||||
$orchestration->remove('appwrite-function-'.$tag['$id'], true);
|
||||
Console::info('Removed container for tag ' . $tag['$id']);
|
||||
} catch (Exception $e) {
|
||||
// Do nothing, we don't care that much if it fails
|
||||
}
|
||||
} catch (Exception $e) {
|
||||
Console::error($e->getMessage());
|
||||
return $response->json(['error' => $e->getMessage()]);
|
||||
}
|
||||
|
||||
return $response->json(['success' => true]);
|
||||
});
|
||||
|
||||
App::post('/v1/tag')
|
||||
->param('functionId', '', new UID(), 'Function unique ID.')
|
||||
->param('tagId', '', new UID(), 'Tag unique ID.')
|
||||
|
@ -132,8 +246,14 @@ App::post('/v1/tag')
|
|||
]));
|
||||
Authorization::reset();
|
||||
|
||||
// Deploy Runtime Server
|
||||
createRuntimeServer($functionId, $projectID, $tag);
|
||||
// Build Code
|
||||
go(function() use ($projectDB, $projectID, $function, $tagId, $functionId) {
|
||||
// Build Code
|
||||
$tag = runBuildStage($tagId, $function, $projectID, $projectDB);
|
||||
|
||||
// Deploy Runtime Server
|
||||
createRuntimeServer($functionId, $projectID, $tag, $projectDB);
|
||||
});
|
||||
|
||||
if ($next) { // Init first schedule
|
||||
ResqueScheduler::enqueueAt($next, 'v1-functions', 'FunctionsV1', [
|
||||
|
@ -165,27 +285,210 @@ App::get('/v1/healthz')
|
|||
}
|
||||
);
|
||||
|
||||
function createRuntimeServer(string $functionId, string $projectId, Document $tag) {
|
||||
function runBuildStage(string $tagID, Document $function, string $projectID, Database $database): Document
|
||||
{
|
||||
global $runtimes;
|
||||
global $orchestration;
|
||||
|
||||
// Update Tag Status
|
||||
Authorization::disable();
|
||||
$tag = $database->getDocument($tagID);
|
||||
$tag = $database->updateDocument(array_merge($tag->getArrayCopy(), [
|
||||
'status' => 'building'
|
||||
]));
|
||||
Authorization::reset();
|
||||
|
||||
// Check if runtime is active
|
||||
$runtime = (isset($runtimes[$function->getAttribute('runtime', '')]))
|
||||
? $runtimes[$function->getAttribute('runtime', '')]
|
||||
: null;
|
||||
|
||||
if ($tag->getAttribute('functionId') !== $function->getId()) {
|
||||
throw new Exception('Tag not found', 404);
|
||||
}
|
||||
|
||||
if (\is_null($runtime)) {
|
||||
throw new Exception('Runtime "' . $function->getAttribute('runtime', '') . '" is not supported');
|
||||
}
|
||||
|
||||
// Grab Tag Files
|
||||
$tagPath = $tag->getAttribute('path', '');
|
||||
$tagPathTarget = '/tmp/project-' . $projectID . '/' . $tag->getId() . '/code.tar.gz';
|
||||
$tagPathTargetDir = \pathinfo($tagPathTarget, PATHINFO_DIRNAME);
|
||||
$container = 'build-stage-' . $tag->getId();
|
||||
|
||||
if (!\is_readable($tagPath)) {
|
||||
throw new Exception('Code is not readable: ' . $tag->getAttribute('path', ''));
|
||||
}
|
||||
|
||||
if (!\file_exists($tagPathTargetDir)) {
|
||||
if (!\mkdir($tagPathTargetDir, 0755, true)) {
|
||||
throw new Exception('Can\'t create directory ' . $tagPathTargetDir);
|
||||
}
|
||||
}
|
||||
|
||||
if (!\file_exists($tagPathTarget)) {
|
||||
if (!\copy($tagPath, $tagPathTarget)) {
|
||||
throw new Exception('Can\'t create temporary code file ' . $tagPathTarget);
|
||||
}
|
||||
}
|
||||
|
||||
$vars = \array_merge($function->getAttribute('vars', []), [
|
||||
'APPWRITE_FUNCTION_ID' => $function->getId(),
|
||||
'APPWRITE_FUNCTION_NAME' => $function->getAttribute('name', ''),
|
||||
'APPWRITE_FUNCTION_TAG' => $tag->getId(),
|
||||
'APPWRITE_FUNCTION_RUNTIME_NAME' => $runtime['name'],
|
||||
'APPWRITE_FUNCTION_RUNTIME_VERSION' => $runtime['version'],
|
||||
'APPWRITE_FUNCTION_PROJECT_ID' => $projectID,
|
||||
]);
|
||||
|
||||
$buildStart = \microtime(true);
|
||||
$buildTime = \time();
|
||||
|
||||
$orchestration->setCpus(App::getEnv('_APP_FUNCTIONS_CPUS', '1'));
|
||||
$orchestration->setMemory(App::getEnv('_APP_FUNCTIONS_MEMORY', '256'));
|
||||
$orchestration->setSwap(App::getEnv('_APP_FUNCTIONS_MEMORY_SWAP', '256'));
|
||||
|
||||
foreach ($vars as &$value) {
|
||||
$value = strval($value);
|
||||
}
|
||||
|
||||
$id = $orchestration->run(
|
||||
image: $runtime['base'],
|
||||
name: $container,
|
||||
vars: $vars,
|
||||
workdir: '/usr/code',
|
||||
labels: [
|
||||
'appwrite-type' => 'function',
|
||||
'appwrite-created' => strval($buildTime),
|
||||
'appwrite-runtime' => $function->getAttribute('runtime', ''),
|
||||
],
|
||||
command: [
|
||||
'tail',
|
||||
'-f',
|
||||
'/dev/null'
|
||||
],
|
||||
hostname: $container,
|
||||
mountFolder: $tagPathTargetDir,
|
||||
volumes: [
|
||||
'/tmp/project-' . $projectID . '/' . $tag->getId() . '/builtCode'. ':/usr/builtCode:rw'
|
||||
]
|
||||
);
|
||||
|
||||
$untarStdout = '';
|
||||
$untarStderr = '';
|
||||
|
||||
$untarSuccess = $orchestration->execute(
|
||||
name: $container,
|
||||
command: [
|
||||
'sh',
|
||||
'-c',
|
||||
'mkdir /usr/code -p && cp /tmp/code.tar.gz /usr/code.tar.gz && cd /usr && tar -zxf /usr/code.tar.gz -C /usr/code && rm /usr/code.tar.gz'
|
||||
],
|
||||
stdout: $untarStdout,
|
||||
stderr: $untarStderr,
|
||||
timeout: 60
|
||||
);
|
||||
|
||||
if (!$untarSuccess) {
|
||||
throw new Exception('Failed to extract tar: ' . $untarStderr);
|
||||
}
|
||||
|
||||
// Build Code / Install Dependencies
|
||||
$buildStdout = '';
|
||||
$buildStderr = '';
|
||||
|
||||
$buildSuccess = $orchestration->execute(
|
||||
name: $container,
|
||||
command: $runtime['buildCommand'],
|
||||
stdout: $buildStdout,
|
||||
stderr: $buildStderr,
|
||||
timeout: 60
|
||||
);
|
||||
|
||||
if (!$buildSuccess) {
|
||||
throw new Exception('Failed to build dependencies: ' . $buildStderr);
|
||||
}
|
||||
|
||||
// Repackage Code and Save.
|
||||
$compressStdout = '';
|
||||
$compressStderr = '';
|
||||
|
||||
$builtCodePath = '/tmp/project-' . $projectID . '/' . $tag->getId() . '/builtCode/code.tar.gz';
|
||||
|
||||
$compressSuccess = $orchestration->execute(
|
||||
name: $container,
|
||||
command: [
|
||||
'sh',
|
||||
'-c',
|
||||
'tar -czvf /usr/builtCode/code.tar.gz /usr/code'
|
||||
],
|
||||
stdout: $compressStdout,
|
||||
stderr: $compressStderr,
|
||||
timeout: 60
|
||||
);
|
||||
|
||||
if (!$compressSuccess) {
|
||||
throw new Exception('Failed to compress built code: ' . $compressStderr);
|
||||
}
|
||||
|
||||
// Remove Container
|
||||
$orchestration->remove($id, true);
|
||||
|
||||
// Check if the build was successful by checking if file exists
|
||||
if (!\file_exists($builtCodePath)) {
|
||||
throw new Exception('Something went wrong during the build process.');
|
||||
}
|
||||
|
||||
// Upload new code
|
||||
$device = Storage::getDevice('functions');
|
||||
|
||||
$path = $device->getPath(\uniqid().'.'.\pathinfo('code.tar.gz', PATHINFO_EXTENSION));
|
||||
|
||||
if (!\file_exists(\dirname($path))) { // Checks if directory path to file exists
|
||||
if (!@\mkdir(\dirname($path), 0755, true)) {
|
||||
throw new Exception('Can\'t create directory: ' . \dirname($path));
|
||||
}
|
||||
}
|
||||
|
||||
if (!\rename($builtCodePath, $path)) {
|
||||
throw new Exception('Failed moving file', 500);
|
||||
}
|
||||
|
||||
// Update tag with built code attribute
|
||||
Authorization::disable();
|
||||
$tag = $database->updateDocument(array_merge($tag->getArrayCopy(), [
|
||||
'builtPath' => $path,
|
||||
'status' => 'ready'
|
||||
]));
|
||||
Authorization::enable();
|
||||
|
||||
$buildEnd = \microtime(true);
|
||||
|
||||
Console::info('Tag Built in ' . ($buildEnd - $buildStart) . ' seconds');
|
||||
|
||||
return $tag;
|
||||
}
|
||||
|
||||
function createRuntimeServer(string $functionId, string $projectId, Document $tag, Database $database)
|
||||
{
|
||||
global $register;
|
||||
global $orchestration;
|
||||
global $runtimes;
|
||||
global $activeFunctions;
|
||||
|
||||
$db = $register->get('db');
|
||||
$cache = $register->get('cache');
|
||||
|
||||
// Create new Database Instance
|
||||
$database = new Database();
|
||||
$database->setAdapter(new RedisAdapter(new MySQLAdapter($db, $cache), $cache));
|
||||
$database->setNamespace('app_' . $projectId);
|
||||
$database->setMocks(Config::getParam('collections', []));
|
||||
|
||||
// Grab Tag Document
|
||||
Authorization::disable();
|
||||
$function = $database->getDocument($functionId);
|
||||
$tag = $database->getDocument($function->getAttribute('tag', ''));
|
||||
Authorization::reset();
|
||||
|
||||
// Check if function isn't already created
|
||||
$functions = $orchestration->list(['label' => 'appwrite-type=function', 'name' => 'appwrite-function-' . $tag->getId()]);
|
||||
|
||||
if (\count($functions) > 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Check if runtime is active
|
||||
$runtime = (isset($runtimes[$function->getAttribute('runtime', '')]))
|
||||
? $runtimes[$function->getAttribute('runtime', '')]
|
||||
|
@ -211,7 +514,7 @@ function createRuntimeServer(string $functionId, string $projectId, Document $ta
|
|||
|
||||
$container = 'appwrite-function-' . $tag->getId();
|
||||
|
||||
if (isset($activeFunctions[$container]) && !(\substr($activeFunctions[$container]->getStatus(), 0, 2) === 'Up')) { // Remove conatiner if not online
|
||||
if (isset($activeFunctions[$container]) && !(\substr($activeFunctions[$container]->getStatus(), 0, 2) === 'Up')) { // Remove container if not online
|
||||
// If container is online then stop and remove it
|
||||
try {
|
||||
$orchestration->remove($container);
|
||||
|
@ -222,8 +525,14 @@ function createRuntimeServer(string $functionId, string $projectId, Document $ta
|
|||
unset($activeFunctions[$container]);
|
||||
}
|
||||
|
||||
// Check if tag is built yet.
|
||||
if ($tag->getAttribute('status') !== 'ready') {
|
||||
throw new Exception('Tag is not built yet', 500);
|
||||
}
|
||||
|
||||
// Grab Tag Files
|
||||
$tagPath = $tag->getAttribute('path', '');
|
||||
$tagPath = $tag->getAttribute('builtPath', '');
|
||||
|
||||
$tagPathTarget = '/tmp/project-' . $projectId . '/' . $tag->getId() . '/code.tar.gz';
|
||||
$tagPathTargetDir = \pathinfo($tagPathTarget, PATHINFO_DIRNAME);
|
||||
$container = 'appwrite-function-' . $tag->getId();
|
||||
|
@ -318,22 +627,13 @@ function createRuntimeServer(string $functionId, string $projectId, Document $ta
|
|||
}
|
||||
};
|
||||
|
||||
function execute(string $trigger, string $projectId, string $executionId, string $functionId, string $event = '', string $eventData = '', string $data = '', array $webhooks = [], string $userId = '', string $jwt = ''): array
|
||||
function execute(string $trigger, string $projectId, string $executionId, string $functionId, Database $database, string $event = '', string $eventData = '', string $data = '', array $webhooks = [], string $userId = '', string $jwt = ''): array
|
||||
{
|
||||
Console::info('Executing function: ' . $functionId);
|
||||
|
||||
global $activeFunctions;
|
||||
global $runtimes;
|
||||
|
||||
global $register;
|
||||
|
||||
$db = $register->get('db');
|
||||
$cache = $register->get('cache');
|
||||
|
||||
// Create new Database Instance
|
||||
$database = new Database();
|
||||
$database->setAdapter(new RedisAdapter(new MySQLAdapter($db, $cache), $cache));
|
||||
$database->setNamespace('app_' . $projectId);
|
||||
$database->setMocks(Config::getParam('collections', []));
|
||||
|
||||
// Grab Tag Document
|
||||
Authorization::disable();
|
||||
$function = $database->getDocument($functionId);
|
||||
|
@ -396,10 +696,41 @@ function execute(string $trigger, string $projectId, string $executionId, string
|
|||
|
||||
$container = 'appwrite-function-' . $tag->getId();
|
||||
|
||||
if (!isset($activeFunctions[$container])) { // Create contianer if not ready
|
||||
createRuntimeServer($functionId, $projectId, $tag);
|
||||
} else {
|
||||
Console::info('Container is ready to run');
|
||||
// Check if code is built
|
||||
|
||||
try {
|
||||
if ($tag->getAttribute('status') !== 'ready') {
|
||||
runBuildStage($tag->getId(), $function, $projectId, $database);
|
||||
sleep(1);
|
||||
}
|
||||
} catch (Exception $e) {
|
||||
Console::error('Something went wrong building the code. ' . $e->getMessage());
|
||||
$execution = $database->updateDocument(array_merge($execution->getArrayCopy(), [
|
||||
'tagId' => $tag->getId(),
|
||||
'status' => 'failed',
|
||||
'exitCode' => 1,
|
||||
'stderr' => \utf8_encode(\mb_substr($e->getMessage(), -4000)), // log last 4000 chars output
|
||||
'time' => 0
|
||||
]));
|
||||
}
|
||||
|
||||
try {
|
||||
if (!isset($activeFunctions[$container])) { // Create contianer if not ready
|
||||
createRuntimeServer($functionId, $projectId, $tag, $database);
|
||||
} else if ($activeFunctions[$container]->getStatus() === 'Down') {
|
||||
sleep(1);
|
||||
} else {
|
||||
Console::info('Container is ready to run');
|
||||
}
|
||||
} catch (Exception $e) {
|
||||
Console::error('Something went wrong building the runtime server. ' . $e->getMessage());
|
||||
$execution = $database->updateDocument(array_merge($execution->getArrayCopy(), [
|
||||
'tagId' => $tag->getId(),
|
||||
'status' => 'failed',
|
||||
'exitCode' => 1,
|
||||
'stderr' => \utf8_encode(\mb_substr($e->getMessage(), -4000)), // log last 4000 chars output
|
||||
'time' => 0
|
||||
]));
|
||||
}
|
||||
|
||||
$stdout = '';
|
||||
|
@ -419,7 +750,7 @@ function execute(string $trigger, string $projectId, string $executionId, string
|
|||
do {
|
||||
$attempts++;
|
||||
$ch = \curl_init();
|
||||
\curl_setopt($ch, CURLOPT_URL, "http://".$container.":3000/");
|
||||
\curl_setopt($ch, CURLOPT_URL, "http://" . $container . ":3000/");
|
||||
\curl_setopt($ch, CURLOPT_POST, true);
|
||||
\curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode([
|
||||
'path' => '/usr/code',
|
||||
|
@ -432,11 +763,11 @@ function execute(string $trigger, string $projectId, string $executionId, string
|
|||
\curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
|
||||
\curl_setopt($ch, CURLOPT_TIMEOUT, $function->getAttribute('timeout', (int) App::getEnv('_APP_FUNCTIONS_TIMEOUT', 900)));
|
||||
\curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 10);
|
||||
|
||||
|
||||
$executorResponse = \curl_exec($ch);
|
||||
|
||||
$error = \curl_error($ch);
|
||||
|
||||
|
||||
$errNo = \curl_errno($ch);
|
||||
|
||||
\curl_close($ch);
|
||||
|
@ -456,7 +787,7 @@ function execute(string $trigger, string $projectId, string $executionId, string
|
|||
if ($errNo == CURLE_OPERATION_TIMEDOUT) {
|
||||
$exitCode = 124;
|
||||
}
|
||||
|
||||
|
||||
if ($errNo !== 0 && $errNo != CURLE_COULDNT_CONNECT && $errNo != CURLE_OPERATION_TIMEDOUT) {
|
||||
throw new Exception('Curl error: ' . $error, 500);
|
||||
}
|
||||
|
@ -487,8 +818,8 @@ function execute(string $trigger, string $projectId, string $executionId, string
|
|||
'tagId' => $tag->getId(),
|
||||
'status' => $functionStatus,
|
||||
'exitCode' => $exitCode,
|
||||
'stdout' => \mb_substr($stdout, -4000), // log last 4000 chars output
|
||||
'stderr' => \mb_substr($stderr, -4000), // log last 4000 chars output
|
||||
'stdout' => \utf8_encode(\mb_substr($stdout, -4000)), // log last 4000 chars output
|
||||
'stderr' => \utf8_encode(\mb_substr($stderr, -4000)), // log last 4000 chars output
|
||||
'time' => $executionTime
|
||||
]));
|
||||
|
||||
|
@ -578,12 +909,14 @@ $http->on('request', function (SwooleRequest $swooleRequest, SwooleResponse $swo
|
|||
|
||||
$projectId = $request->getHeader('x-appwrite-project', '');
|
||||
|
||||
App::setResource('projectDB', function($db, $cache) use ($projectId) {
|
||||
Storage::setDevice('functions', new Local(APP_STORAGE_FUNCTIONS.'/app-'.$projectId));
|
||||
|
||||
App::setResource('projectDB', function ($db, $cache) use ($projectId) {
|
||||
$projectDB = new Database();
|
||||
$projectDB->setAdapter(new RedisAdapter(new MySQLAdapter($db, $cache), $cache));
|
||||
$projectDB->setNamespace('app_'.$projectId);
|
||||
$projectDB->setNamespace('app_' . $projectId);
|
||||
$projectDB->setMocks(Config::getParam('collections', []));
|
||||
|
||||
|
||||
return $projectDB;
|
||||
}, ['db', 'cache']);
|
||||
|
||||
|
@ -592,31 +925,31 @@ $http->on('request', function (SwooleRequest $swooleRequest, SwooleResponse $swo
|
|||
/** @var Utopia\App $utopia */
|
||||
/** @var Utopia\Swoole\Request $request */
|
||||
/** @var Appwrite\Utopia\Response $response */
|
||||
|
||||
|
||||
if ($error instanceof PDOException) {
|
||||
throw $error;
|
||||
}
|
||||
|
||||
|
||||
$route = $utopia->match($request);
|
||||
|
||||
Console::error('[Error] Timestamp: '.date('c', time()));
|
||||
|
||||
if($route) {
|
||||
Console::error('[Error] Method: '.$route->getMethod());
|
||||
|
||||
Console::error('[Error] Timestamp: ' . date('c', time()));
|
||||
|
||||
if ($route) {
|
||||
Console::error('[Error] Method: ' . $route->getMethod());
|
||||
}
|
||||
|
||||
Console::error('[Error] Type: '.get_class($error));
|
||||
Console::error('[Error] Message: '.$error->getMessage());
|
||||
Console::error('[Error] File: '.$error->getFile());
|
||||
Console::error('[Error] Line: '.$error->getLine());
|
||||
|
||||
|
||||
Console::error('[Error] Type: ' . get_class($error));
|
||||
Console::error('[Error] Message: ' . $error->getMessage());
|
||||
Console::error('[Error] File: ' . $error->getFile());
|
||||
Console::error('[Error] Line: ' . $error->getLine());
|
||||
|
||||
$version = App::getEnv('_APP_VERSION', 'UNKNOWN');
|
||||
|
||||
$code = $error->getCode();
|
||||
$message = $error->getMessage();
|
||||
|
||||
|
||||
//$_SERVER = []; // Reset before reporting to error log to avoid keys being compromised
|
||||
|
||||
|
||||
$output = ((App::isDevelopment())) ? [
|
||||
'message' => $error->getMessage(),
|
||||
'code' => $error->getCode(),
|
||||
|
@ -629,19 +962,20 @@ $http->on('request', function (SwooleRequest $swooleRequest, SwooleResponse $swo
|
|||
'code' => $code,
|
||||
'version' => $version,
|
||||
];
|
||||
|
||||
|
||||
$response
|
||||
->addHeader('Cache-Control', 'no-cache, no-store, must-revalidate')
|
||||
->addHeader('Expires', '0')
|
||||
->addHeader('Pragma', 'no-cache')
|
||||
->setStatusCode($code)
|
||||
;
|
||||
|
||||
$response->dynamic(new Document($output),
|
||||
$utopia->isDevelopment() ? Response::MODEL_ERROR_DEV : Response::MODEL_ERROR);
|
||||
->setStatusCode($code);
|
||||
|
||||
$response->dynamic(
|
||||
new Document($output),
|
||||
$utopia->isDevelopment() ? Response::MODEL_ERROR_DEV : Response::MODEL_ERROR
|
||||
);
|
||||
}, ['error', 'utopia', 'request', 'response']);
|
||||
|
||||
App::setResource('projectID', function() use ($projectId) {
|
||||
App::setResource('projectID', function () use ($projectId) {
|
||||
return $projectId;
|
||||
});
|
||||
|
||||
|
@ -655,7 +989,8 @@ $http->on('request', function (SwooleRequest $swooleRequest, SwooleResponse $swo
|
|||
|
||||
$http->start();
|
||||
|
||||
function handleShutdown() {
|
||||
function handleShutdown()
|
||||
{
|
||||
Console::info('Cleaning up containers before shutdown...');
|
||||
|
||||
// Remove all containers.
|
||||
|
@ -666,9 +1001,9 @@ function handleShutdown() {
|
|||
foreach ($functionsToRemove as $container) {
|
||||
try {
|
||||
$orchestration->remove($container->getId(), true);
|
||||
Console::info('Removed container '.$container->getName());
|
||||
Console::info('Removed container ' . $container->getName());
|
||||
} catch (Exception $e) {
|
||||
Console::error('Failed to remove container: '.$container->getName());
|
||||
Console::error('Failed to remove container: ' . $container->getName());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -34,27 +34,6 @@ $dockerPass = App::getEnv('DOCKERHUB_PULL_PASSWORD', null);
|
|||
$dockerEmail = App::getEnv('DOCKERHUB_PULL_EMAIL', null);
|
||||
$orchestration = new Orchestration(new DockerAPI($dockerUser, $dockerPass, $dockerEmail));
|
||||
|
||||
/**
|
||||
* Warmup Docker Images
|
||||
*/
|
||||
$warmupStart = \microtime(true);
|
||||
|
||||
Co\run(function () use ($runtimes, $orchestration) { // Warmup: make sure images are ready to run fast 🚀
|
||||
foreach ($runtimes as $runtime) {
|
||||
go(function () use ($runtime, $orchestration) {
|
||||
Console::info('Warming up ' . $runtime['name'] . ' ' . $runtime['version'] . ' environment...');
|
||||
|
||||
$response = $orchestration->pull($runtime['image']);
|
||||
|
||||
if ($response) {
|
||||
Console::success("Successfully Warmed up {$runtime['name']} {$runtime['version']}!");
|
||||
} else {
|
||||
Console::error("Failed to Warmup {$runtime['name']} {$runtime['version']}!");
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
$warmupEnd = \microtime(true);
|
||||
$warmupTime = $warmupEnd - $warmupStart;
|
||||
|
||||
|
|
|
@ -66,7 +66,6 @@
|
|||
"utopia-php/orchestration": "dev-exp1",
|
||||
"resque/php-resque": "1.3.6",
|
||||
"matomo/device-detector": "4.2.3",
|
||||
"utopia-php/orchestration": "0.2.1",
|
||||
"dragonmantank/cron-expression": "3.1.0",
|
||||
"influxdb/influxdb-php": "1.15.2",
|
||||
"phpmailer/phpmailer": "6.5.0",
|
||||
|
|
5
composer.lock
generated
5
composer.lock
generated
|
@ -119,7 +119,7 @@
|
|||
"source": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/PineappleIOnic/php-runtimes.git",
|
||||
"reference": "d633a896c4e5c20fd166f4b5461a869b0db2616e"
|
||||
"reference": "147e76f4a72bc925d9d1613494417d583bd5de34"
|
||||
},
|
||||
"require": {
|
||||
"php": ">=8.0",
|
||||
|
@ -155,7 +155,7 @@
|
|||
"php",
|
||||
"runtimes"
|
||||
],
|
||||
"time": "2021-09-02T09:13:23+00:00"
|
||||
"time": "2021-09-06T14:46:02+00:00"
|
||||
},
|
||||
{
|
||||
"name": "chillerlan/php-qrcode",
|
||||
|
@ -4920,6 +4920,7 @@
|
|||
"type": "github"
|
||||
}
|
||||
],
|
||||
"abandoned": true,
|
||||
"time": "2020-09-28T06:45:17+00:00"
|
||||
},
|
||||
{
|
||||
|
|
|
@ -368,7 +368,7 @@ services:
|
|||
entrypoint:
|
||||
- php
|
||||
- -e
|
||||
- app/executor.php
|
||||
- /usr/src/code/app/executor.php
|
||||
- -dopcache.preload=opcache.preload=/usr/src/code/app/preload.php
|
||||
stop_signal: SIGINT
|
||||
ports:
|
||||
|
@ -376,7 +376,7 @@ services:
|
|||
build:
|
||||
context: .
|
||||
args:
|
||||
- DEBUG=false
|
||||
- DEBUG=true
|
||||
- TESTING=true
|
||||
- VERSION=dev
|
||||
networks:
|
||||
|
|
|
@ -186,7 +186,7 @@ class FunctionsCustomClientTest extends Scope
|
|||
$this->assertEquals(201, $execution['headers']['status-code']);
|
||||
|
||||
$executionId = $execution['body']['$id'] ?? '';
|
||||
|
||||
|
||||
sleep(10);
|
||||
|
||||
$executions = $this->client->call(Client::METHOD_GET, '/functions/'.$functionId.'/executions/'.$executionId, [
|
||||
|
|
|
@ -492,6 +492,9 @@ class FunctionsCustomServerTest extends Scope
|
|||
]);
|
||||
|
||||
$this->assertEquals(200, $tag['headers']['status-code']);
|
||||
|
||||
// Allow build step to run
|
||||
sleep(20);
|
||||
|
||||
$execution = $this->client->call(Client::METHOD_POST, '/functions/'.$functionId.'/executions', array_merge([
|
||||
'content-type' => 'application/json',
|
||||
|
|
Binary file not shown.
|
@ -13,5 +13,6 @@ return function ($request, $response) {
|
|||
'APPWRITE_FUNCTION_DATA' => $request->env['APPWRITE_FUNCTION_DATA'],
|
||||
'APPWRITE_FUNCTION_USER_ID' => $request->env['APPWRITE_FUNCTION_USER_ID'],
|
||||
'APPWRITE_FUNCTION_JWT' => $request->env['APPWRITE_FUNCTION_JWT'],
|
||||
'APPWRITE_FUNCTION_PROJECT_ID' => $request->env['APPWRITE_FUNCTION_PROJECT_ID'],
|
||||
]);
|
||||
};
|
||||
|
|
Binary file not shown.
Loading…
Reference in a new issue