CodeMirror/appwrite
1
0
Fork 0
mirror of synced 2024-05-10 15:52:31 +12:00
Code Issues Projects Releases Wiki Activity

Updated usage tracking vars

Browse source
This commit is contained in:
Eldad Fux 2020-07-20 09:43:25 +03:00
parent 11ccdba946
commit b9c848f531
3 changed files with 51 additions and 40 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
app/app.php
View file

@ -235,10 +235,11 @@ App::init(function ($utopia, $request, $response, $console, $project, $user, $lo
$usage $usage
->setParam('projectId', $project->getId()) ->setParam('projectId', $project->getId())
->setParam('url', $request->getHostname().$request->getURI()) ->setParam('httpRequest', 1)
->setParam('method', $request->getMethod()) ->setParam('httpUrl', $request->getHostname().$request->getURI())
->setParam('request', 0) ->setParam('httpMethod', $request->getMethod())
->setParam('response', 0) ->setParam('networkRequestSize', 0)
->setParam('networkResponseSize', 0)
->setParam('storage', 0) ->setParam('storage', 0)
; ;
}, ['utopia', 'request', 'response', 'console', 'project', 'user', 'locale', 'webhooks', 'audits', 'usage', 'clients']); }, ['utopia', 'request', 'response', 'console', 'project', 'user', 'locale', 'webhooks', 'audits', 'usage', 'clients']);
@ -273,8 +274,8 @@ App::shutdown(function ($utopia, $request, $response, $project, $webhooks, $audi
&& !empty($route->getLabel('sdk.namespace', null))) { // Don't calculate console usage and admin mode && !empty($route->getLabel('sdk.namespace', null))) { // Don't calculate console usage and admin mode
$usage $usage
->setParam('request', $request->getSize() + $usage->getParam('storage')) ->setParam('networkRequestSize', $request->getSize() + $usage->getParam('storage'))
->setParam('response', $response->getSize()) ->setParam('networkResponseSize', $response->getSize())
->trigger() ->trigger()
; ;
} }

50
app/workers/functions.php
View file

@ -13,13 +13,13 @@ use Utopia\Config\Config;
require_once __DIR__.'/../init.php'; require_once __DIR__.'/../init.php';
cli_set_process_title('Functions V1 Worker'); \cli_set_process_title('Functions V1 Worker');
Console::success(APP_NAME.' functions worker v1 has started'); Console::success(APP_NAME.' functions worker v1 has started');
$environments = Config::getParam('environments'); $environments = Config::getParam('environments');
$warmupStart = microtime(true); $warmupStart = \microtime(true);
Co\run(function() use ($environments) { Co\run(function() use ($environments) {
foreach($environments as $environment) { // Warmup: make sure images are ready to run fast 🚀 foreach($environments as $environment) { // Warmup: make sure images are ready to run fast 🚀
@ -42,7 +42,7 @@ Co\run(function() use ($environments) {
} }
}); });
$warmupEnd = microtime(true); $warmupEnd = \microtime(true);
$warmupTime = $warmupEnd - $warmupStart; $warmupTime = $warmupEnd - $warmupStart;
Console::success('Finished warmup in '.$warmupTime.' seconds'); Console::success('Finished warmup in '.$warmupTime.' seconds');
@ -67,8 +67,8 @@ Console::success('Finished warmup in '.$warmupTime.' seconds');
* + messure execution time - DONE * + messure execution time - DONE
* + pass env vars - DONE * + pass env vars - DONE
* + pass one-time api key * + pass one-time api key
* 4. Update execution status * 4. Update execution status - DONE
* 5. Update execution stdout & stderr * 5. Update execution stdout & stderr - DONE
* 6. Trigger audit log * 6. Trigger audit log
* 7. Trigger usage log * 7. Trigger usage log
*/ */
@ -145,7 +145,7 @@ class FunctionsV1
'read' => [], 'read' => [],
'write' => [], 'write' => [],
], ],
'dateCreated' => time(), 'dateCreated' => \time(),
'functionId' => $function->getId(), 'functionId' => $function->getId(),
'status' => 'processing', // waiting / processing / completed / failed 'status' => 'processing', // waiting / processing / completed / failed
'exitCode' => 0, 'exitCode' => 0,
@ -169,7 +169,7 @@ class FunctionsV1
throw new Exception('Environment "'.$function->getAttribute('env', '').' is not supported'); throw new Exception('Environment "'.$function->getAttribute('env', '').' is not supported');
} }
$vars = array_merge($function->getAttribute('vars', []), [ $vars = \array_merge($function->getAttribute('vars', []), [
'APPWRITE_FUNCTION_ID' => $functionId, 'APPWRITE_FUNCTION_ID' => $functionId,
'APPWRITE_FUNCTION_NAME' => $function->getAttribute('name', ''), 'APPWRITE_FUNCTION_NAME' => $function->getAttribute('name', ''),
'APPWRITE_FUNCTION_TAG' => $functionTag, 'APPWRITE_FUNCTION_TAG' => $functionTag,
@ -178,7 +178,7 @@ class FunctionsV1
'APPWRITE_FUNCTION_ENV_VERSION' => $environment['version'], 'APPWRITE_FUNCTION_ENV_VERSION' => $environment['version'],
]); ]);
array_walk($vars, function (&$value, $key) { \array_walk($vars, function (&$value, $key) {
$value = (empty($value)) ? 'null' : $value; $value = (empty($value)) ? 'null' : $value;
$value = "\t\t\t--env {$key}={$value} \\"; $value = "\t\t\t--env {$key}={$value} \\";
}); });
@ -187,7 +187,7 @@ class FunctionsV1
$tagPathTarget = '/tmp/project-'.$projectId.'/'.$tag->getId().'/code.tar.gz'; $tagPathTarget = '/tmp/project-'.$projectId.'/'.$tag->getId().'/code.tar.gz';
$tagPathTargetDir = \pathinfo($tagPathTarget, PATHINFO_DIRNAME); $tagPathTargetDir = \pathinfo($tagPathTarget, PATHINFO_DIRNAME);
$container = 'appwrite-function-'.$tag->getId(); $container = 'appwrite-function-'.$tag->getId();
$command = escapeshellcmd($tag->getAttribute('command', '')); $command = \escapeshellcmd($tag->getAttribute('command', ''));
if(!\is_readable($tagPath)) { if(!\is_readable($tagPath)) {
throw new Exception('Code is not readable: '.$tag->getAttribute('codePath', '')); throw new Exception('Code is not readable: '.$tag->getAttribute('codePath', ''));
@ -216,28 +216,28 @@ class FunctionsV1
$executionEnd = \microtime(true); $executionEnd = \microtime(true);
$list = []; $list = [];
$stdout = explode("\n", $stdout); $stdout = \explode("\n", $stdout);
array_map(function($value) use (&$list) { \array_map(function($value) use (&$list) {
$container = []; $container = [];
parse_str($value, $container); \parse_str($value, $container);
if(isset($container['name'])) { if(isset($container['name'])) {
$container = [ $container = [
'name' => $container['name'], 'name' => $container['name'],
'online' => (substr($container['status'], 0, 2) === 'Up'), 'online' => (\substr($container['status'], 0, 2) === 'Up'),
'status' => $container['status'], 'status' => $container['status'],
'labels' => $container['labels'], 'labels' => $container['labels'],
]; ];
array_map(function($value) use (&$container) { \array_map(function($value) use (&$container) {
$value = explode('=', $value); $value = \explode('=', $value);
if(isset($value[0]) && isset($value[1])) { if(isset($value[0]) && isset($value[1])) {
$container[$value[0]] = $value[1]; $container[$value[0]] = $value[1];
} }
}, explode(',', $container['labels'])); }, \explode(',', $container['labels']));
$list[$container['name']] = $container; $list[$container['name']] = $container;
} }
@ -245,8 +245,6 @@ class FunctionsV1
Console::info("Functions listed in " . ($executionEnd - $executionStart) . " seconds with exit code {$exitCode}"); Console::info("Functions listed in " . ($executionEnd - $executionStart) . " seconds with exit code {$exitCode}");
//TODO if container found and offline -> delete it
if(isset($list[$container]) && !$list[$container]['online']) { if(isset($list[$container]) && !$list[$container]['online']) {
$stdout = ''; $stdout = '';
$stderr = ''; $stderr = '';
@ -258,8 +256,6 @@ class FunctionsV1
unset($list[$container]); unset($list[$container]);
} }
//TODO if more than x contianers -> clear all older containers
if(!isset($list[$container])) { // Create contianer if not ready if(!isset($list[$container])) { // Create contianer if not ready
$stdout = ''; $stdout = '';
$stderr = ''; $stderr = '';
@ -275,10 +271,10 @@ class FunctionsV1
--rm \ --rm \
--name={$container} \ --name={$container} \
--label appwrite-type=function \ --label appwrite-type=function \
--label appwrite-created=".time()." \ --label appwrite-created=".\time()." \
--volume {$tagPathTargetDir}:/tmp:rw \ --volume {$tagPathTargetDir}:/tmp:rw \
--workdir /usr/local/src \ --workdir /usr/local/src \
".implode("\n", $vars)." ".\implode("\n", $vars)."
{$environment['image']} \ {$environment['image']} \
sh -c 'mv /tmp/code.tar.gz /usr/local/src/code.tar.gz && tar -zxf /usr/local/src/code.tar.gz --strip 1 && rm /usr/local/src/code.tar.gz && tail -f /dev/null'" sh -c 'mv /tmp/code.tar.gz /usr/local/src/code.tar.gz && tar -zxf /usr/local/src/code.tar.gz --strip 1 && rm /usr/local/src/code.tar.gz && tail -f /dev/null'"
, null, $stdout, $stderr, 30); , null, $stdout, $stderr, 30);
@ -301,7 +297,7 @@ class FunctionsV1
$executionStart = \microtime(true); $executionStart = \microtime(true);
$exitCode = Console::execute("docker exec {$container} {$command}" $exitCode = Console::execute("docker exec {$container} {$command}"
, null, $stdout, $stderr, $function->getAttribute('timeout', (int) App::getEnv('_APP_FUNCTIONS_TIMEOUT', 900))); // TODO add app env for max timeout , null, $stdout, $stderr, $function->getAttribute('timeout', (int) App::getEnv('_APP_FUNCTIONS_TIMEOUT', 900)));
$executionEnd = \microtime(true); $executionEnd = \microtime(true);
@ -328,7 +324,7 @@ class FunctionsV1
$max = (int) App::getEnv('_APP_FUNCTIONS_CONTAINERS'); $max = (int) App::getEnv('_APP_FUNCTIONS_CONTAINERS');
if(count($list) > $max) { if(\count($list) > $max) {
Console::info('Starting containers cleanup'); Console::info('Starting containers cleanup');
$sorted = []; $sorted = [];
@ -340,12 +336,12 @@ class FunctionsV1
]; ];
} }
usort($sorted, function ($item1, $item2) { \usort($sorted, function ($item1, $item2) {
return $item1['created'] <=> $item2['created']; return $item1['created'] <=> $item2['created'];
}); });
while(count($sorted) > $max) { while(\count($sorted) > $max) {
$first = array_shift($sorted); $first = \array_shift($sorted);
$stdout = ''; $stdout = '';
$stderr = ''; $stderr = '';

28
app/workers/usage.php
View file

@ -27,21 +27,35 @@ class UsageV1
$statsd = $register->get('statsd', true); $statsd = $register->get('statsd', true);
$projectId = $this->args['projectId']; $projectId = $this->args['projectId'];
$method = $this->args['method']; $httpMethod = $this->args['httpMethod'];
$request = $this->args['request']; $httpRequest = $this->args['httpRequest'];
$response = $this->args['response'];
$networkRequestSize = $this->args['networkRequestSize'];
$networkResponseSize = $this->args['networkResponseSize'];
$storage = $this->args['storage']; $storage = $this->args['storage'];
$functionExecution = $this->args['functionExecution'];
$functionExecutionTime = $this->args['functionExecutionTime'];
$functionId = $this->args['functionId'];
$tags = ",project={$projectId},version=".App::getEnv('_APP_VERSION', 'UNKNOWN').''; $tags = ",project={$projectId},version=".App::getEnv('_APP_VERSION', 'UNKNOWN').'';
// the global namespace is prepended to every key (optional) // the global namespace is prepended to every key (optional)
$statsd->setNamespace('appwrite.usage'); $statsd->setNamespace('appwrite.usage');
$statsd->increment('requests.all'.$tags.',method='.\strtolower($method)); if($httpRequest >= 1) {
$statsd->increment('requests.all'.$tags.',method='.\strtolower($httpMethod));
}
if($functionExecution >= 1) {
$statsd->increment('executions.all'.$tags.',functionId='.$functionId);
$statsd->count('executions.time'.$tags.',functionId='.$functionId, $functionExecutionTime);
}
$statsd->count('network.all'.$tags, $request + $response); $statsd->count('network.all'.$tags, $networkRequestSize + $networkResponseSize);
$statsd->count('network.inbound'.$tags, $request); $statsd->count('network.inbound'.$tags, $networkRequestSize);
$statsd->count('network.outbound'.$tags, $response); $statsd->count('network.outbound'.$tags, $networkResponseSize);
$statsd->count('storage.all'.$tags, $storage); $statsd->count('storage.all'.$tags, $storage);
} }