CodeMirror/appwrite
1
0
Fork 0
mirror of synced 2024-09-21 03:52:23 +12:00
Code Issues Projects Releases Wiki Activity

Merge pull request #1457 from appwrite/feat-depricate-usage-worker

Browse source 
feat-depricate-usage-worker
This commit is contained in:
Eldad A. Fux 2021-08-09 08:18:40 +03:00 committed by GitHub
commit 328071a8e6
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 224 additions and 124 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
Dockerfile
View file

@ -238,7 +238,6 @@ RUN chmod +x /usr/local/bin/doctor && \
chmod +x /usr/local/bin/worker-deletes && \
chmod +x /usr/local/bin/worker-functions && \
chmod +x /usr/local/bin/worker-mails && \
chmod +x /usr/local/bin/worker-usage && \
chmod +x /usr/local/bin/worker-webhooks
# Letsencrypt Permissions

10
app/controllers/shared/api.php
View file

@ -18,7 +18,7 @@ App::init(function ($utopia, $request, $response, $project, $user, $register, $e
/** @var Utopia\Registry\Registry $register */
/** @var Appwrite\Event\Event $events */
/** @var Appwrite\Event\Event $audits */
/** @var Appwrite\Event\Event $usage */
/** @var Appwrite\Stats\Stats $usage */
/** @var Appwrite\Event\Event $deletes */
/** @var Appwrite\Event\Event $database */
/** @var Appwrite\Event\Event $functions */
@ -162,14 +162,14 @@ App::init(function ($utopia, $request, $project) {
}, ['utopia', 'request', 'project'], 'auth');
App::shutdown(function ($utopia, $request, $response, $project, $events, $audits, $usage, $deletes, $database, $mode) {
App::shutdown(function ($utopia, $request, $response, $project, $register, $events, $audits, $usage, $deletes, $database, $mode) {
/** @var Utopia\App $utopia */
/** @var Utopia\Swoole\Request $request */
/** @var Appwrite\Utopia\Response $response */
/** @var Utopia\Database\Document $project */
/** @var Appwrite\Event\Event $events */
/** @var Appwrite\Event\Event $audits */
/** @var Appwrite\Event\Event $usage */
/** @var Appwrite\Stats\Stats $usage */
/** @var Appwrite\Event\Event $deletes */
/** @var Appwrite\Event\Event $database */
/** @var Appwrite\Event\Event $functions */
@ -215,8 +215,8 @@ App::shutdown(function ($utopia, $request, $response, $project, $events, $audits
$usage
->setParam('networkRequestSize', $request->getSize() + $usage->getParam('storage'))
->setParam('networkResponseSize', $response->getSize())
->trigger()
->submit()
;
}
}, ['utopia', 'request', 'response', 'project', 'events', 'audits', 'usage', 'deletes', 'database', 'mode'], 'api');
}, ['utopia', 'request', 'response', 'project', 'register', 'events', 'audits', 'usage', 'deletes', 'database', 'mode'], 'api');

4
app/init.php
View file

@ -29,6 +29,7 @@ use Appwrite\Network\Validator\Email;
use Appwrite\Network\Validator\IP;
use Appwrite\Network\Validator\URL;
use Appwrite\OpenSSL\OpenSSL;
use Appwrite\Stats\Stats;
use Utopia\App;
use Utopia\View;
use Utopia\Config\Config;
@ -291,6 +292,7 @@ $register->set('statsd', function () { // Register DB connection
return $statsd;
});
$register->set('smtp', function () {
$mail = new PHPMailer(true);
@ -421,7 +423,7 @@ App::setResource('audits', function($register) {
}, ['register']);
App::setResource('usage', function($register) {
return new Event(Event::USAGE_QUEUE_NAME, Event::USAGE_CLASS_NAME);
return new Stats($register->get('statsd'));
}, ['register']);
App::setResource('mails', function($register) {

32
app/workers/functions.php
View file

@ -2,6 +2,7 @@
use Appwrite\Event\Event;
use Appwrite\Resque\Worker;
use Appwrite\Stats\Stats;
use Appwrite\Utopia\Response\Model\Execution;
use Cron\CronExpression;
use Swoole\Runtime;
@ -134,8 +135,6 @@ class FunctionsV1 extends Worker
public function run(): void
{
global $register;
$projectId = $this->args['projectId'] ?? '';
$functionId = $this->args['functionId'] ?? '';
$webhooks = $this->args['webhooks'] ?? [];
@ -279,7 +278,7 @@ class FunctionsV1 extends Worker
*/
public function execute(string $trigger, string $projectId, string $executionId, Database $database, Document $function, string $event = '', string $eventData = '', string $data = '', array $webhooks = [], string $userId = '', string $jwt = ''): void
{
global $list;
global $list, $register;
$runtimes = Config::getParam('runtimes');
@ -477,21 +476,22 @@ class FunctionsV1 extends Worker
->setParam('eventData', $execution->getArrayCopy(array_keys($executionModel->getRules())));
$executionUpdate->trigger();
$usage = new Event('v1-usage', 'UsageV1');
$usage
->setParam('projectId', $projectId)
->setParam('functionId', $function->getId())
->setParam('functionExecution', 1)
->setParam('functionStatus', $functionStatus)
->setParam('functionExecutionTime', $executionTime * 1000) // ms
->setParam('networkRequestSize', 0)
->setParam('networkResponseSize', 0)
;
if(App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') {
$usage->trigger();
$statsd = $register->get('statsd');
$usage = new Stats($statsd);
$usage
->setParam('projectId', $projectId)
->setParam('functionId', $function->getId())
->setParam('functionExecution', 1)
->setParam('functionStatus', $functionStatus)
->setParam('functionExecutionTime', $executionTime * 1000) // ms
->setParam('networkRequestSize', 0)
->setParam('networkResponseSize', 0)
->submit()
;
}
$this->cleanup();

71
app/workers/usage.php
View file

@ -1,71 +0,0 @@
<?php
use Appwrite\Resque\Worker;
use Utopia\App;
use Utopia\CLI\Console;
require_once __DIR__.'/../workers.php';
Console::title('Usage V1 Worker');
Console::success(APP_NAME.' usage worker v1 has started');
class UsageV1 extends Worker
{
/**
* @var array
*/
public $args = [];
public function init(): void
{
}
public function run(): void
{
global $register;
/** @var \Domnikl\Statsd\Client $statsd */
$statsd = $register->get('statsd', true);
$projectId = $this->args['projectId'] ?? '';
$storage = $this->args['storage'] ?? 0;
$networkRequestSize = $this->args['networkRequestSize'] ?? 0;
$networkResponseSize = $this->args['networkResponseSize'] ?? 0;
$httpMethod = $this->args['httpMethod'] ?? '';
$httpRequest = $this->args['httpRequest'] ?? 0;
$functionId = $this->args['functionId'] ?? '';
$functionExecution = $this->args['functionExecution'] ?? 0;
$functionExecutionTime = $this->args['functionExecutionTime'] ?? 0;
$functionStatus = $this->args['functionStatus'] ?? '';
$tags = ",project={$projectId},version=".App::getEnv('_APP_VERSION', 'UNKNOWN');
// the global namespace is prepended to every key (optional)
$statsd->setNamespace('appwrite.usage');
if($httpRequest >= 1) {
$statsd->increment('requests.all'.$tags.',method='.\strtolower($httpMethod));
}
if($functionExecution >= 1) {
$statsd->increment('executions.all'.$tags.',functionId='.$functionId.',functionStatus='.$functionStatus);
$statsd->count('executions.time'.$tags.',functionId='.$functionId, $functionExecutionTime);
}
$statsd->count('network.inbound'.$tags, $networkRequestSize);
$statsd->count('network.outbound'.$tags, $networkResponseSize);
$statsd->count('network.all'.$tags, $networkRequestSize + $networkResponseSize);
if($storage >= 1) {
$statsd->count('storage.all'.$tags, $storage);
}
}
public function shutdown(): void
{
}
}

10
bin/worker-usage
View file

@ -1,10 +0,0 @@
#!/bin/sh
if [ -z "$_APP_REDIS_USER" ] && [ -z "$_APP_REDIS_PASS" ]
then
REDIS_BACKEND="${_APP_REDIS_HOST}:${_APP_REDIS_PORT}"
else
REDIS_BACKEND="redis://${_APP_REDIS_USER}:${_APP_REDIS_PASS}@${_APP_REDIS_HOST}:${_APP_REDIS_PORT}"
fi
INTERVAL=1 QUEUE='v1-usage' APP_INCLUDE='/usr/src/code/app/workers/usage.php' php /usr/src/code/vendor/bin/resque -dopcache.preload=opcache.preload=/usr/src/code/app/preload.php

22
docker-compose.yml
View file

@ -121,26 +121,6 @@ services:
- _APP_FUNCTIONS_MEMORY
- _APP_FUNCTIONS_MEMORY_SWAP
- _APP_FUNCTIONS_RUNTIMES
appwrite-worker-usage:
entrypoint: worker-usage
container_name: appwrite-worker-usage
build:
context: .
networks:
- appwrite
volumes:
- ./app:/usr/src/code/app
- ./src:/usr/src/code/src
depends_on:
- redis
- telegraf
environment:
- _APP_ENV
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_STATSD_HOST
- _APP_STATSD_PORT
@ -308,6 +288,8 @@ services:
- _APP_FUNCTIONS_MEMORY
- _APP_FUNCTIONS_MEMORY_SWAP
- _APP_USAGE_STATS
- _APP_STATSD_HOST
- _APP_STATSD_PORT
- DOCKERHUB_PULL_USERNAME
- DOCKERHUB_PULL_PASSWORD

129
src/Appwrite/Stats/Stats.php Normal file
View file

@ -0,0 +1,129 @@
<?php
namespace Appwrite\Stats;
use Utopia\App;
class Stats
{
/**
* @var array
*/
protected $params = [];
/**
* @var mixed
*/
protected $statsd;
/**
* @var string
*/
protected $namespace = 'appwrite.usage';
/**
* Event constructor.
*
* @param mixed $statsd
*/
public function __construct($statsd)
{
$this->statsd = $statsd;
}
/**
* @param string $key
* @param mixed $value
*
* @return $this
*/
public function setParam(string $key, $value): self
{
$this->params[$key] = $value;
return $this;
}
/**
* @param string $key
*
* @return mixed|null
*/
public function getParam(string $key)
{
return (isset($this->params[$key])) ? $this->params[$key] : null;
}
/**
* @param string $namespace
*
* @return $this
*/
public function setNamespace(string $namespace): self
{
$this->namespace = $namespace;
return $this;
}
/**
* @return string
*/
public function getNamespace()
{
return $this->namespace;
}
/**
* Submit data to StatsD.
*/
public function submit(): void
{
$projectId = $this->params['projectId'] ?? '';
$storage = $this->params['storage'] ?? 0;
$networkRequestSize = $this->params['networkRequestSize'] ?? 0;
$networkResponseSize = $this->params['networkResponseSize'] ?? 0;
$httpMethod = $this->params['httpMethod'] ?? '';
$httpRequest = $this->params['httpRequest'] ?? 0;
$functionId = $this->params['functionId'] ?? '';
$functionExecution = $this->params['functionExecution'] ?? 0;
$functionExecutionTime = $this->params['functionExecutionTime'] ?? 0;
$functionStatus = $this->params['functionStatus'] ?? '';
$tags = ",project={$projectId},version=" . App::getEnv('_APP_VERSION', 'UNKNOWN');
// the global namespace is prepended to every key (optional)
$this->statsd->setNamespace($this->namespace);
if ($httpRequest >= 1) {
$this->statsd->increment('requests.all' . $tags . ',method=' . \strtolower($httpMethod));
}
if ($functionExecution >= 1) {
$this->statsd->increment('executions.all' . $tags . ',functionId=' . $functionId . ',functionStatus=' . $functionStatus);
$this->statsd->count('executions.time' . $tags . ',functionId=' . $functionId, $functionExecutionTime);
}
$this->statsd->count('network.inbound' . $tags, $networkRequestSize);
$this->statsd->count('network.outbound' . $tags, $networkResponseSize);
$this->statsd->count('network.all' . $tags, $networkRequestSize + $networkResponseSize);
if ($storage >= 1) {
$this->statsd->count('storage.all' . $tags, $storage);
}
$this->reset();
}
public function reset(): self
{
$this->params = [];
$this->namespace = 'appwrite.usage';
return $this;
}
}

69
tests/unit/Stats/StatsTest.php Normal file
View file

@ -0,0 +1,69 @@
<?php
namespace Appwrite\Tests;
use Appwrite\Stats\Stats;
use PHPUnit\Framework\TestCase;
use Utopia\App;
class StatsTest extends TestCase
{
/**
* @var Stats
*/
protected $object = null;
public function setUp(): void
{
$host = App::getEnv('_APP_STATSD_HOST', 'telegraf');
$port = App::getEnv('_APP_STATSD_PORT', 8125);
$connection = new \Domnikl\Statsd\Connection\UdpSocket($host, $port);
$statsd = new \Domnikl\Statsd\Client($connection);
$this->object = new Stats($statsd);
}
public function tearDown(): void
{
}
public function testNamespace()
{
$this->object->setNamespace('appwritetest.usage');
$this->assertEquals('appwritetest.usage', $this->object->getNamespace());
}
public function testParams()
{
$this->object
->setParam('projectId', 'appwrite_test')
->setParam('networkRequestSize', 100)
;
$this->assertEquals('appwrite_test', $this->object->getParam('projectId'));
$this->assertEquals(100, $this->object->getParam('networkRequestSize'));
$this->object->submit();
$this->assertEquals(null, $this->object->getParam('projectId'));
$this->assertEquals(null, $this->object->getParam('networkRequestSize'));
}
public function testReset()
{
$this->object
->setParam('projectId', 'appwrite_test')
->setParam('networkRequestSize', 100)
;
$this->assertEquals('appwrite_test', $this->object->getParam('projectId'));
$this->assertEquals(100, $this->object->getParam('networkRequestSize'));
$this->object->reset();
$this->assertEquals(null, $this->object->getParam('projectId'));
$this->assertEquals(null, $this->object->getParam('networkRequestSize'));
$this->assertEquals('appwrite.usage', $this->object->getNamespace());
}
}