feat: maintenance task
This commit is contained in:
parent
ff5314e202
commit
0acd573c89
3 changed files with 66 additions and 56 deletions
1
.env
1
.env
|
@ -48,3 +48,4 @@ _APP_MAINTENANCE_RETENTION_AUDIT=1209600
|
||||||
_APP_USAGE_STATS=enabled
|
_APP_USAGE_STATS=enabled
|
||||||
_APP_LOGGING_PROVIDER=
|
_APP_LOGGING_PROVIDER=
|
||||||
_APP_LOGGING_CONFIG=
|
_APP_LOGGING_CONFIG=
|
||||||
|
OPENRUNTIMES_INACTIVE_THRESHOLD=60
|
120
app/executor.php
120
app/executor.php
|
@ -32,7 +32,8 @@ use Utopia\Validator\Text;
|
||||||
// Pull runtimes on startup -- Done
|
// Pull runtimes on startup -- Done
|
||||||
// Move some logic to server start - Done
|
// Move some logic to server start - Done
|
||||||
// Add updated property to swoole table - Done
|
// Add updated property to swoole table - Done
|
||||||
// Clean up deployments older than X seconds
|
// Clean up deployments older than X seconds - Done
|
||||||
|
|
||||||
// Remove attempts logic in executor
|
// Remove attempts logic in executor
|
||||||
// Fix delete endpoint
|
// Fix delete endpoint
|
||||||
// Remove orphans on startup
|
// Remove orphans on startup
|
||||||
|
@ -45,17 +46,20 @@ use Utopia\Validator\Text;
|
||||||
|
|
||||||
Runtime::enableCoroutine(SWOOLE_HOOK_ALL);
|
Runtime::enableCoroutine(SWOOLE_HOOK_ALL);
|
||||||
|
|
||||||
|
/** Constants */
|
||||||
|
const MAINTENANCE_INTERVAL = 1200; // 20 minutes
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Create a Swoole table to store runtime information
|
* Create a Swoole table to store runtime information
|
||||||
*/
|
*/
|
||||||
$activeFunctions = new Swoole\Table(1024);
|
$activeRuntimes = new Swoole\Table(1024);
|
||||||
$activeFunctions->column('id', Swoole\Table::TYPE_STRING, 512);
|
$activeRuntimes->column('id', Swoole\Table::TYPE_STRING, 512);
|
||||||
$activeFunctions->column('created', Swoole\Table::TYPE_INT, 8);
|
$activeRuntimes->column('created', Swoole\Table::TYPE_INT, 8);
|
||||||
$activeFunctions->column('updated', Swoole\Table::TYPE_INT, 8);
|
$activeRuntimes->column('updated', Swoole\Table::TYPE_INT, 8);
|
||||||
$activeFunctions->column('name', Swoole\Table::TYPE_STRING, 512);
|
$activeRuntimes->column('name', Swoole\Table::TYPE_STRING, 512);
|
||||||
$activeFunctions->column('status', Swoole\Table::TYPE_STRING, 512);
|
$activeRuntimes->column('status', Swoole\Table::TYPE_STRING, 512);
|
||||||
$activeFunctions->column('key', Swoole\Table::TYPE_STRING, 512);
|
$activeRuntimes->column('key', Swoole\Table::TYPE_STRING, 512);
|
||||||
$activeFunctions->create();
|
$activeRuntimes->create();
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Create orchestration pool
|
* Create orchestration pool
|
||||||
|
@ -120,9 +124,9 @@ App::post('/v1/runtimes')
|
||||||
->param('runtime', '', new Text(128), 'Runtime for the cloud function')
|
->param('runtime', '', new Text(128), 'Runtime for the cloud function')
|
||||||
->param('baseImage', '', new Text(128), 'Base image name of the runtime')
|
->param('baseImage', '', new Text(128), 'Base image name of the runtime')
|
||||||
->inject('orchestrationPool')
|
->inject('orchestrationPool')
|
||||||
->inject('activeFunctions')
|
->inject('activeRuntimes')
|
||||||
->inject('response')
|
->inject('response')
|
||||||
->action(function (string $runtimeId, string $source, string $destination, array $vars, string $runtime, string $baseImage, $orchestrationPool, $activeFunctions, Response $response) {
|
->action(function (string $runtimeId, string $source, string $destination, array $vars, string $runtime, string $baseImage, $orchestrationPool, $activeRuntimes, Response $response) {
|
||||||
|
|
||||||
// TODO: Check if runtime already exists..
|
// TODO: Check if runtime already exists..
|
||||||
$orchestration = $orchestrationPool->get();
|
$orchestration = $orchestrationPool->get();
|
||||||
|
@ -319,14 +323,14 @@ App::post('/v1/runtimes')
|
||||||
/** Create runtime server */
|
/** Create runtime server */
|
||||||
try {
|
try {
|
||||||
$container = 'runtime-' . $runtimeId;
|
$container = 'runtime-' . $runtimeId;
|
||||||
if ($activeFunctions->exists($container) && !(\substr($activeFunctions->get($container)['status'], 0, 2) === 'Up')) { // Remove container if not online
|
if ($activeRuntimes->exists($container) && !(\substr($activeRuntimes->get($container)['status'], 0, 2) === 'Up')) { // Remove container if not online
|
||||||
// If container is online then stop and remove it
|
// If container is online then stop and remove it
|
||||||
try {
|
try {
|
||||||
$orchestration->remove($container, true);
|
$orchestration->remove($container, true);
|
||||||
} catch (Exception $e) {
|
} catch (Exception $e) {
|
||||||
throw new Exception('Failed to remove container: ' . $e->getMessage());
|
throw new Exception('Failed to remove container: ' . $e->getMessage());
|
||||||
}
|
}
|
||||||
$activeFunctions->del($container);
|
$activeRuntimes->del($container);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -345,7 +349,7 @@ App::post('/v1/runtimes')
|
||||||
'INTERNAL_RUNTIME_KEY' => $secret
|
'INTERNAL_RUNTIME_KEY' => $secret
|
||||||
]);
|
]);
|
||||||
|
|
||||||
if (!$activeFunctions->exists($container)) {
|
if (!$activeRuntimes->exists($container)) {
|
||||||
$executionStart = \microtime(true);
|
$executionStart = \microtime(true);
|
||||||
$executionTime = \time();
|
$executionTime = \time();
|
||||||
|
|
||||||
|
@ -379,7 +383,7 @@ App::post('/v1/runtimes')
|
||||||
|
|
||||||
$executionEnd = \microtime(true);
|
$executionEnd = \microtime(true);
|
||||||
|
|
||||||
$activeFunctions->set($container, [
|
$activeRuntimes->set($container, [
|
||||||
'id' => $id,
|
'id' => $id,
|
||||||
'name' => $container,
|
'name' => $container,
|
||||||
'created' => $executionTime,
|
'created' => $executionTime,
|
||||||
|
@ -404,12 +408,12 @@ App::post('/v1/runtimes')
|
||||||
// GET /v1/runtimes
|
// GET /v1/runtimes
|
||||||
App::get('/v1/runtimes')
|
App::get('/v1/runtimes')
|
||||||
->desc("List currently active runtimes")
|
->desc("List currently active runtimes")
|
||||||
->inject('activeFunctions')
|
->inject('activeRuntimes')
|
||||||
->inject('response')
|
->inject('response')
|
||||||
->action(function ($activeFunctions, Response $response) {
|
->action(function ($activeRuntimes, Response $response) {
|
||||||
$runtimes = [];
|
$runtimes = [];
|
||||||
|
|
||||||
foreach($activeFunctions as $runtime) {
|
foreach($activeRuntimes as $runtime) {
|
||||||
$runtimes[] = $runtime;
|
$runtimes[] = $runtime;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -423,15 +427,15 @@ App::get('/v1/runtimes')
|
||||||
App::get('/v1/runtimes/:runtimeId')
|
App::get('/v1/runtimes/:runtimeId')
|
||||||
->desc("Get a runtime by its ID")
|
->desc("Get a runtime by its ID")
|
||||||
->param('runtimeId', '', new Text(128), 'Runtime unique ID.')
|
->param('runtimeId', '', new Text(128), 'Runtime unique ID.')
|
||||||
->inject('activeFunctions')
|
->inject('activeRuntimes')
|
||||||
->inject('response')
|
->inject('response')
|
||||||
->action(function ($runtimeId, $activeFunctions, Response $response) {
|
->action(function ($runtimeId, $activeRuntimes, Response $response) {
|
||||||
|
|
||||||
if(!$activeFunctions->exists($runtimeId)) {
|
if(!$activeRuntimes->exists($runtimeId)) {
|
||||||
throw new Exception('Runtime not found', 404);
|
throw new Exception('Runtime not found', 404);
|
||||||
}
|
}
|
||||||
|
|
||||||
$runtime = $activeFunctions->get($runtimeId);
|
$runtime = $activeRuntimes->get($runtimeId);
|
||||||
|
|
||||||
$response
|
$response
|
||||||
->setStatusCode(200)
|
->setStatusCode(200)
|
||||||
|
@ -444,20 +448,23 @@ App::delete('/v1/runtimes/:runtimeId')
|
||||||
->param('runtimeId', '', new Text(128), 'Runtime unique ID.', false)
|
->param('runtimeId', '', new Text(128), 'Runtime unique ID.', false)
|
||||||
->param('buildIds', [], new ArrayList(new Text(0), 100), 'List of build IDs to delete.', false)
|
->param('buildIds', [], new ArrayList(new Text(0), 100), 'List of build IDs to delete.', false)
|
||||||
->inject('orchestrationPool')
|
->inject('orchestrationPool')
|
||||||
|
->inject('activeRuntimes')
|
||||||
->inject('response')
|
->inject('response')
|
||||||
->action(function (string $runtimeId, array $buildIds, $orchestrationPool, Response $response) {
|
->action(function (string $runtimeId, array $buildIds, $orchestrationPool, $activeRuntimes, Response $response) {
|
||||||
|
|
||||||
Console::info('Deleting runtime: ' . $runtimeId);
|
Console::info('Deleting runtime: ' . $runtimeId);
|
||||||
$orchestration = $orchestrationPool->get();
|
$orchestration = $orchestrationPool->get();
|
||||||
|
|
||||||
// Remove the container of the deployment
|
$container = 'runtime-' . $runtimeId;
|
||||||
$status = $orchestration->remove('runtime-' . $runtimeId , true);
|
$status = $orchestration->remove($container, true);
|
||||||
if ($status) {
|
if ($status) {
|
||||||
Console::success('Removed runtime container: ' . $runtimeId);
|
Console::success('Removed runtime container: ' . $runtimeId);
|
||||||
} else {
|
} else {
|
||||||
Console::error('Failed to remove runtime container: ' . $runtimeId);
|
Console::error('Failed to remove runtime container: ' . $runtimeId);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
$activeRuntimes->del($container);
|
||||||
|
|
||||||
// Remove all the build containers with that same ID
|
// Remove all the build containers with that same ID
|
||||||
// TODO:: Delete build containers
|
// TODO:: Delete build containers
|
||||||
// foreach ($buildIds as $buildId) {
|
// foreach ($buildIds as $buildId) {
|
||||||
|
@ -487,19 +494,19 @@ App::post('/v1/execution')
|
||||||
->param('entrypoint', '', new Text(256), 'Entrypoint of the code file')
|
->param('entrypoint', '', new Text(256), 'Entrypoint of the code file')
|
||||||
->param('timeout', 15, new ValidatorRange(1, 900), 'Function maximum execution time in seconds.', true)
|
->param('timeout', 15, new ValidatorRange(1, 900), 'Function maximum execution time in seconds.', true)
|
||||||
->param('baseImage', '', new Text(128), 'Base image name of the runtime', false)
|
->param('baseImage', '', new Text(128), 'Base image name of the runtime', false)
|
||||||
->inject('activeFunctions')
|
->inject('activeRuntimes')
|
||||||
->inject('response')
|
->inject('response')
|
||||||
->action(
|
->action(
|
||||||
function (string $runtimeId, string $path, array $vars, string $data, string $runtime, string $entrypoint, $timeout, string $baseImage, $activeFunctions, Response $response) {
|
function (string $runtimeId, string $path, array $vars, string $data, string $runtime, string $entrypoint, $timeout, string $baseImage, $activeRuntimes, Response $response) {
|
||||||
|
|
||||||
$container = 'runtime-' . $runtimeId;
|
$container = 'runtime-' . $runtimeId;
|
||||||
|
|
||||||
// TODO: Also check for container status
|
// TODO: Also check for container status
|
||||||
if (!$activeFunctions->exists($container)) {
|
if (!$activeRuntimes->exists($container)) {
|
||||||
throw new Exception('Runtime not found. Please create the runtime.', 404);
|
throw new Exception('Runtime not found. Please create the runtime.', 404);
|
||||||
}
|
}
|
||||||
|
|
||||||
$runtime = $activeFunctions->get($container);
|
$runtime = $activeRuntimes->get($container);
|
||||||
$secret = $runtime['key'];
|
$secret = $runtime['key'];
|
||||||
if (empty($secret)) {
|
if (empty($secret)) {
|
||||||
throw new Exception('Runtime secret not found. Please create the runtime.', 500);
|
throw new Exception('Runtime secret not found. Please create the runtime.', 500);
|
||||||
|
@ -617,7 +624,7 @@ App::post('/v1/execution')
|
||||||
];
|
];
|
||||||
|
|
||||||
$runtime['updated'] = \time();
|
$runtime['updated'] = \time();
|
||||||
$activeFunctions->set($container, $runtime);
|
$activeRuntimes->set($container, $runtime);
|
||||||
} catch (\Throwable $th) {
|
} catch (\Throwable $th) {
|
||||||
Console::error('Runtime execution failed: ' . $th->getMessage());
|
Console::error('Runtime execution failed: ' . $th->getMessage());
|
||||||
}
|
}
|
||||||
|
@ -634,7 +641,7 @@ $http = new Server("0.0.0.0", 80);
|
||||||
|
|
||||||
/** Set Resources */
|
/** Set Resources */
|
||||||
App::setResource('orchestrationPool', fn() => $orchestrationPool);
|
App::setResource('orchestrationPool', fn() => $orchestrationPool);
|
||||||
App::setResource('activeFunctions', fn() => $activeFunctions);
|
App::setResource('activeRuntimes', fn() => $activeRuntimes);
|
||||||
|
|
||||||
/** Set callbacks */
|
/** Set callbacks */
|
||||||
App::error(function ($error, $utopia, $request, $response) {
|
App::error(function ($error, $utopia, $request, $response) {
|
||||||
|
@ -717,7 +724,7 @@ $http->on('start', function ($http) {
|
||||||
/**
|
/**
|
||||||
* Populate swoole table with active runtimes
|
* Populate swoole table with active runtimes
|
||||||
*/
|
*/
|
||||||
global $activeFunctions;
|
global $activeRuntimes;
|
||||||
try {
|
try {
|
||||||
$orchestration = $orchestrationPool->get();
|
$orchestration = $orchestrationPool->get();
|
||||||
$executionStart = \microtime(true);
|
$executionStart = \microtime(true);
|
||||||
|
@ -728,7 +735,7 @@ $http->on('start', function ($http) {
|
||||||
}
|
}
|
||||||
|
|
||||||
foreach ($residueList as $value) {
|
foreach ($residueList as $value) {
|
||||||
go(fn () => $activeFunctions->set($value->getName(), [
|
go(fn () => $activeRuntimes->set($value->getName(), [
|
||||||
'id' => $value->getId(),
|
'id' => $value->getId(),
|
||||||
'name' => $value->getName(),
|
'name' => $value->getName(),
|
||||||
'status' => $value->getStatus(),
|
'status' => $value->getStatus(),
|
||||||
|
@ -737,7 +744,7 @@ $http->on('start', function ($http) {
|
||||||
}
|
}
|
||||||
|
|
||||||
$executionEnd = \microtime(true);
|
$executionEnd = \microtime(true);
|
||||||
Console::info(count($activeFunctions) . ' functions listed in ' . ($executionEnd - $executionStart) . ' seconds');
|
Console::info(count($activeRuntimes) . ' functions listed in ' . ($executionEnd - $executionStart) . ' seconds');
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Register handlers for shutdown
|
* Register handlers for shutdown
|
||||||
|
@ -759,29 +766,30 @@ $http->on('start', function ($http) {
|
||||||
});
|
});
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Run a maintenance worker every 5 minutes to remove unused containers
|
* Run a maintenance worker every MAINTENANCE_INTERVAL seconds to remove inactive runtimes
|
||||||
*/
|
*/
|
||||||
// global $orchestrationPool;
|
global $orchestrationPool;
|
||||||
// Timer::tick(5000, function () use ($orchestrationPool) {
|
global $activeRuntimes;
|
||||||
// Console::info('Running maintenance task every 5 seconds ...');
|
Timer::tick(MAINTENANCE_INTERVAL * 1000, function () use ($orchestrationPool, $activeRuntimes) {
|
||||||
// $orchestration = $orchestrationPool->get();
|
Console::warning("Running maintenance task ...");
|
||||||
// $functionsToRemove = $orchestration->list(['label' => 'openruntimes-type=function']);
|
foreach ($activeRuntimes as $runtime) {
|
||||||
// $orchestrationPool->put($orchestration);
|
$inactiveThreshold = \time() - App::getEnv('OPENRUNTIMES_INACTIVE_THRESHOLD', 60);
|
||||||
|
if ($runtime['updated'] < $inactiveThreshold) {
|
||||||
// foreach ($functionsToRemove as $container) {
|
go(function () use ($runtime, $orchestrationPool, $activeRuntimes) {
|
||||||
// go(function () use ($orchestrationPool, $container) {
|
try {
|
||||||
// try {
|
$orchestration = $orchestrationPool->get();
|
||||||
// $orchestration = $orchestrationPool->get();
|
$orchestration->remove($runtime['name'], true);
|
||||||
// $orchestration->remove($container->getId(), true);
|
$activeRuntimes->del($runtime['name']);
|
||||||
// Console::info('Removed container ' . $container->getName());
|
Console::success("Successfully removed {$runtime['name']}");
|
||||||
// } catch (\Throwable $th) {
|
} catch (\Throwable $th) {
|
||||||
// Console::error('Failed to remove container: ' . $container->getName());
|
Console::error('Inactive Runtime deletion failed: ' . $th->getMessage());
|
||||||
// } finally {
|
} finally {
|
||||||
// $orchestrationPool->put($orchestration);
|
$orchestrationPool->put($orchestration);
|
||||||
// }
|
}
|
||||||
// });
|
});
|
||||||
// }
|
}
|
||||||
// });
|
}
|
||||||
|
});
|
||||||
|
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
|
@ -416,6 +416,7 @@ services:
|
||||||
- _APP_LOGGING_CONFIG
|
- _APP_LOGGING_CONFIG
|
||||||
- DOCKERHUB_PULL_USERNAME
|
- DOCKERHUB_PULL_USERNAME
|
||||||
- DOCKERHUB_PULL_PASSWORD
|
- DOCKERHUB_PULL_PASSWORD
|
||||||
|
- OPENRUNTIMES_INACTIVE_THRESHOLD
|
||||||
|
|
||||||
appwrite-worker-mails:
|
appwrite-worker-mails:
|
||||||
entrypoint: worker-mails
|
entrypoint: worker-mails
|
||||||
|
|
Loading…
Reference in a new issue