Merge pull request #1457 from appwrite/feat-depricate-usage-worker
feat-depricate-usage-worker
This commit is contained in:
commit
328071a8e6
9 changed files with 224 additions and 124 deletions
|
@ -238,7 +238,6 @@ RUN chmod +x /usr/local/bin/doctor && \
|
||||||
chmod +x /usr/local/bin/worker-deletes && \
|
chmod +x /usr/local/bin/worker-deletes && \
|
||||||
chmod +x /usr/local/bin/worker-functions && \
|
chmod +x /usr/local/bin/worker-functions && \
|
||||||
chmod +x /usr/local/bin/worker-mails && \
|
chmod +x /usr/local/bin/worker-mails && \
|
||||||
chmod +x /usr/local/bin/worker-usage && \
|
|
||||||
chmod +x /usr/local/bin/worker-webhooks
|
chmod +x /usr/local/bin/worker-webhooks
|
||||||
|
|
||||||
# Letsencrypt Permissions
|
# Letsencrypt Permissions
|
||||||
|
|
|
@ -18,7 +18,7 @@ App::init(function ($utopia, $request, $response, $project, $user, $register, $e
|
||||||
/** @var Utopia\Registry\Registry $register */
|
/** @var Utopia\Registry\Registry $register */
|
||||||
/** @var Appwrite\Event\Event $events */
|
/** @var Appwrite\Event\Event $events */
|
||||||
/** @var Appwrite\Event\Event $audits */
|
/** @var Appwrite\Event\Event $audits */
|
||||||
/** @var Appwrite\Event\Event $usage */
|
/** @var Appwrite\Stats\Stats $usage */
|
||||||
/** @var Appwrite\Event\Event $deletes */
|
/** @var Appwrite\Event\Event $deletes */
|
||||||
/** @var Appwrite\Event\Event $database */
|
/** @var Appwrite\Event\Event $database */
|
||||||
/** @var Appwrite\Event\Event $functions */
|
/** @var Appwrite\Event\Event $functions */
|
||||||
|
@ -162,14 +162,14 @@ App::init(function ($utopia, $request, $project) {
|
||||||
|
|
||||||
}, ['utopia', 'request', 'project'], 'auth');
|
}, ['utopia', 'request', 'project'], 'auth');
|
||||||
|
|
||||||
App::shutdown(function ($utopia, $request, $response, $project, $events, $audits, $usage, $deletes, $database, $mode) {
|
App::shutdown(function ($utopia, $request, $response, $project, $register, $events, $audits, $usage, $deletes, $database, $mode) {
|
||||||
/** @var Utopia\App $utopia */
|
/** @var Utopia\App $utopia */
|
||||||
/** @var Utopia\Swoole\Request $request */
|
/** @var Utopia\Swoole\Request $request */
|
||||||
/** @var Appwrite\Utopia\Response $response */
|
/** @var Appwrite\Utopia\Response $response */
|
||||||
/** @var Utopia\Database\Document $project */
|
/** @var Utopia\Database\Document $project */
|
||||||
/** @var Appwrite\Event\Event $events */
|
/** @var Appwrite\Event\Event $events */
|
||||||
/** @var Appwrite\Event\Event $audits */
|
/** @var Appwrite\Event\Event $audits */
|
||||||
/** @var Appwrite\Event\Event $usage */
|
/** @var Appwrite\Stats\Stats $usage */
|
||||||
/** @var Appwrite\Event\Event $deletes */
|
/** @var Appwrite\Event\Event $deletes */
|
||||||
/** @var Appwrite\Event\Event $database */
|
/** @var Appwrite\Event\Event $database */
|
||||||
/** @var Appwrite\Event\Event $functions */
|
/** @var Appwrite\Event\Event $functions */
|
||||||
|
@ -215,8 +215,8 @@ App::shutdown(function ($utopia, $request, $response, $project, $events, $audits
|
||||||
$usage
|
$usage
|
||||||
->setParam('networkRequestSize', $request->getSize() + $usage->getParam('storage'))
|
->setParam('networkRequestSize', $request->getSize() + $usage->getParam('storage'))
|
||||||
->setParam('networkResponseSize', $response->getSize())
|
->setParam('networkResponseSize', $response->getSize())
|
||||||
->trigger()
|
->submit()
|
||||||
;
|
;
|
||||||
}
|
}
|
||||||
|
|
||||||
}, ['utopia', 'request', 'response', 'project', 'events', 'audits', 'usage', 'deletes', 'database', 'mode'], 'api');
|
}, ['utopia', 'request', 'response', 'project', 'register', 'events', 'audits', 'usage', 'deletes', 'database', 'mode'], 'api');
|
|
@ -29,6 +29,7 @@ use Appwrite\Network\Validator\Email;
|
||||||
use Appwrite\Network\Validator\IP;
|
use Appwrite\Network\Validator\IP;
|
||||||
use Appwrite\Network\Validator\URL;
|
use Appwrite\Network\Validator\URL;
|
||||||
use Appwrite\OpenSSL\OpenSSL;
|
use Appwrite\OpenSSL\OpenSSL;
|
||||||
|
use Appwrite\Stats\Stats;
|
||||||
use Utopia\App;
|
use Utopia\App;
|
||||||
use Utopia\View;
|
use Utopia\View;
|
||||||
use Utopia\Config\Config;
|
use Utopia\Config\Config;
|
||||||
|
@ -291,6 +292,7 @@ $register->set('statsd', function () { // Register DB connection
|
||||||
|
|
||||||
return $statsd;
|
return $statsd;
|
||||||
});
|
});
|
||||||
|
|
||||||
$register->set('smtp', function () {
|
$register->set('smtp', function () {
|
||||||
$mail = new PHPMailer(true);
|
$mail = new PHPMailer(true);
|
||||||
|
|
||||||
|
@ -421,7 +423,7 @@ App::setResource('audits', function($register) {
|
||||||
}, ['register']);
|
}, ['register']);
|
||||||
|
|
||||||
App::setResource('usage', function($register) {
|
App::setResource('usage', function($register) {
|
||||||
return new Event(Event::USAGE_QUEUE_NAME, Event::USAGE_CLASS_NAME);
|
return new Stats($register->get('statsd'));
|
||||||
}, ['register']);
|
}, ['register']);
|
||||||
|
|
||||||
App::setResource('mails', function($register) {
|
App::setResource('mails', function($register) {
|
||||||
|
|
|
@ -2,6 +2,7 @@
|
||||||
|
|
||||||
use Appwrite\Event\Event;
|
use Appwrite\Event\Event;
|
||||||
use Appwrite\Resque\Worker;
|
use Appwrite\Resque\Worker;
|
||||||
|
use Appwrite\Stats\Stats;
|
||||||
use Appwrite\Utopia\Response\Model\Execution;
|
use Appwrite\Utopia\Response\Model\Execution;
|
||||||
use Cron\CronExpression;
|
use Cron\CronExpression;
|
||||||
use Swoole\Runtime;
|
use Swoole\Runtime;
|
||||||
|
@ -134,8 +135,6 @@ class FunctionsV1 extends Worker
|
||||||
|
|
||||||
public function run(): void
|
public function run(): void
|
||||||
{
|
{
|
||||||
global $register;
|
|
||||||
|
|
||||||
$projectId = $this->args['projectId'] ?? '';
|
$projectId = $this->args['projectId'] ?? '';
|
||||||
$functionId = $this->args['functionId'] ?? '';
|
$functionId = $this->args['functionId'] ?? '';
|
||||||
$webhooks = $this->args['webhooks'] ?? [];
|
$webhooks = $this->args['webhooks'] ?? [];
|
||||||
|
@ -279,7 +278,7 @@ class FunctionsV1 extends Worker
|
||||||
*/
|
*/
|
||||||
public function execute(string $trigger, string $projectId, string $executionId, Database $database, Document $function, string $event = '', string $eventData = '', string $data = '', array $webhooks = [], string $userId = '', string $jwt = ''): void
|
public function execute(string $trigger, string $projectId, string $executionId, Database $database, Document $function, string $event = '', string $eventData = '', string $data = '', array $webhooks = [], string $userId = '', string $jwt = ''): void
|
||||||
{
|
{
|
||||||
global $list;
|
global $list, $register;
|
||||||
|
|
||||||
$runtimes = Config::getParam('runtimes');
|
$runtimes = Config::getParam('runtimes');
|
||||||
|
|
||||||
|
@ -478,7 +477,10 @@ class FunctionsV1 extends Worker
|
||||||
|
|
||||||
$executionUpdate->trigger();
|
$executionUpdate->trigger();
|
||||||
|
|
||||||
$usage = new Event('v1-usage', 'UsageV1');
|
if(App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') {
|
||||||
|
$statsd = $register->get('statsd');
|
||||||
|
|
||||||
|
$usage = new Stats($statsd);
|
||||||
|
|
||||||
$usage
|
$usage
|
||||||
->setParam('projectId', $projectId)
|
->setParam('projectId', $projectId)
|
||||||
|
@ -488,10 +490,8 @@ class FunctionsV1 extends Worker
|
||||||
->setParam('functionExecutionTime', $executionTime * 1000) // ms
|
->setParam('functionExecutionTime', $executionTime * 1000) // ms
|
||||||
->setParam('networkRequestSize', 0)
|
->setParam('networkRequestSize', 0)
|
||||||
->setParam('networkResponseSize', 0)
|
->setParam('networkResponseSize', 0)
|
||||||
|
->submit()
|
||||||
;
|
;
|
||||||
|
|
||||||
if(App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') {
|
|
||||||
$usage->trigger();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
$this->cleanup();
|
$this->cleanup();
|
||||||
|
|
|
@ -1,71 +0,0 @@
|
||||||
<?php
|
|
||||||
|
|
||||||
use Appwrite\Resque\Worker;
|
|
||||||
use Utopia\App;
|
|
||||||
use Utopia\CLI\Console;
|
|
||||||
|
|
||||||
require_once __DIR__.'/../workers.php';
|
|
||||||
|
|
||||||
Console::title('Usage V1 Worker');
|
|
||||||
Console::success(APP_NAME.' usage worker v1 has started');
|
|
||||||
|
|
||||||
class UsageV1 extends Worker
|
|
||||||
{
|
|
||||||
/**
|
|
||||||
* @var array
|
|
||||||
*/
|
|
||||||
public $args = [];
|
|
||||||
|
|
||||||
public function init(): void
|
|
||||||
{
|
|
||||||
}
|
|
||||||
|
|
||||||
public function run(): void
|
|
||||||
{
|
|
||||||
global $register;
|
|
||||||
|
|
||||||
/** @var \Domnikl\Statsd\Client $statsd */
|
|
||||||
$statsd = $register->get('statsd', true);
|
|
||||||
|
|
||||||
$projectId = $this->args['projectId'] ?? '';
|
|
||||||
|
|
||||||
$storage = $this->args['storage'] ?? 0;
|
|
||||||
|
|
||||||
$networkRequestSize = $this->args['networkRequestSize'] ?? 0;
|
|
||||||
$networkResponseSize = $this->args['networkResponseSize'] ?? 0;
|
|
||||||
|
|
||||||
$httpMethod = $this->args['httpMethod'] ?? '';
|
|
||||||
$httpRequest = $this->args['httpRequest'] ?? 0;
|
|
||||||
|
|
||||||
$functionId = $this->args['functionId'] ?? '';
|
|
||||||
$functionExecution = $this->args['functionExecution'] ?? 0;
|
|
||||||
$functionExecutionTime = $this->args['functionExecutionTime'] ?? 0;
|
|
||||||
$functionStatus = $this->args['functionStatus'] ?? '';
|
|
||||||
|
|
||||||
$tags = ",project={$projectId},version=".App::getEnv('_APP_VERSION', 'UNKNOWN');
|
|
||||||
|
|
||||||
// the global namespace is prepended to every key (optional)
|
|
||||||
$statsd->setNamespace('appwrite.usage');
|
|
||||||
|
|
||||||
if($httpRequest >= 1) {
|
|
||||||
$statsd->increment('requests.all'.$tags.',method='.\strtolower($httpMethod));
|
|
||||||
}
|
|
||||||
|
|
||||||
if($functionExecution >= 1) {
|
|
||||||
$statsd->increment('executions.all'.$tags.',functionId='.$functionId.',functionStatus='.$functionStatus);
|
|
||||||
$statsd->count('executions.time'.$tags.',functionId='.$functionId, $functionExecutionTime);
|
|
||||||
}
|
|
||||||
|
|
||||||
$statsd->count('network.inbound'.$tags, $networkRequestSize);
|
|
||||||
$statsd->count('network.outbound'.$tags, $networkResponseSize);
|
|
||||||
$statsd->count('network.all'.$tags, $networkRequestSize + $networkResponseSize);
|
|
||||||
|
|
||||||
if($storage >= 1) {
|
|
||||||
$statsd->count('storage.all'.$tags, $storage);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public function shutdown(): void
|
|
||||||
{
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -1,10 +0,0 @@
|
||||||
#!/bin/sh
|
|
||||||
|
|
||||||
if [ -z "$_APP_REDIS_USER" ] && [ -z "$_APP_REDIS_PASS" ]
|
|
||||||
then
|
|
||||||
REDIS_BACKEND="${_APP_REDIS_HOST}:${_APP_REDIS_PORT}"
|
|
||||||
else
|
|
||||||
REDIS_BACKEND="redis://${_APP_REDIS_USER}:${_APP_REDIS_PASS}@${_APP_REDIS_HOST}:${_APP_REDIS_PORT}"
|
|
||||||
fi
|
|
||||||
|
|
||||||
INTERVAL=1 QUEUE='v1-usage' APP_INCLUDE='/usr/src/code/app/workers/usage.php' php /usr/src/code/vendor/bin/resque -dopcache.preload=opcache.preload=/usr/src/code/app/preload.php
|
|
|
@ -121,26 +121,6 @@ services:
|
||||||
- _APP_FUNCTIONS_MEMORY
|
- _APP_FUNCTIONS_MEMORY
|
||||||
- _APP_FUNCTIONS_MEMORY_SWAP
|
- _APP_FUNCTIONS_MEMORY_SWAP
|
||||||
- _APP_FUNCTIONS_RUNTIMES
|
- _APP_FUNCTIONS_RUNTIMES
|
||||||
|
|
||||||
appwrite-worker-usage:
|
|
||||||
entrypoint: worker-usage
|
|
||||||
container_name: appwrite-worker-usage
|
|
||||||
build:
|
|
||||||
context: .
|
|
||||||
networks:
|
|
||||||
- appwrite
|
|
||||||
volumes:
|
|
||||||
- ./app:/usr/src/code/app
|
|
||||||
- ./src:/usr/src/code/src
|
|
||||||
depends_on:
|
|
||||||
- redis
|
|
||||||
- telegraf
|
|
||||||
environment:
|
|
||||||
- _APP_ENV
|
|
||||||
- _APP_REDIS_HOST
|
|
||||||
- _APP_REDIS_PORT
|
|
||||||
- _APP_REDIS_USER
|
|
||||||
- _APP_REDIS_PASS
|
|
||||||
- _APP_STATSD_HOST
|
- _APP_STATSD_HOST
|
||||||
- _APP_STATSD_PORT
|
- _APP_STATSD_PORT
|
||||||
|
|
||||||
|
@ -308,6 +288,8 @@ services:
|
||||||
- _APP_FUNCTIONS_MEMORY
|
- _APP_FUNCTIONS_MEMORY
|
||||||
- _APP_FUNCTIONS_MEMORY_SWAP
|
- _APP_FUNCTIONS_MEMORY_SWAP
|
||||||
- _APP_USAGE_STATS
|
- _APP_USAGE_STATS
|
||||||
|
- _APP_STATSD_HOST
|
||||||
|
- _APP_STATSD_PORT
|
||||||
- DOCKERHUB_PULL_USERNAME
|
- DOCKERHUB_PULL_USERNAME
|
||||||
- DOCKERHUB_PULL_PASSWORD
|
- DOCKERHUB_PULL_PASSWORD
|
||||||
|
|
||||||
|
|
129
src/Appwrite/Stats/Stats.php
Normal file
129
src/Appwrite/Stats/Stats.php
Normal file
|
@ -0,0 +1,129 @@
|
||||||
|
<?php
|
||||||
|
|
||||||
|
namespace Appwrite\Stats;
|
||||||
|
|
||||||
|
use Utopia\App;
|
||||||
|
|
||||||
|
class Stats
|
||||||
|
{
|
||||||
|
/**
|
||||||
|
* @var array
|
||||||
|
*/
|
||||||
|
protected $params = [];
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @var mixed
|
||||||
|
*/
|
||||||
|
protected $statsd;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @var string
|
||||||
|
*/
|
||||||
|
protected $namespace = 'appwrite.usage';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Event constructor.
|
||||||
|
*
|
||||||
|
* @param mixed $statsd
|
||||||
|
*/
|
||||||
|
public function __construct($statsd)
|
||||||
|
{
|
||||||
|
$this->statsd = $statsd;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param string $key
|
||||||
|
* @param mixed $value
|
||||||
|
*
|
||||||
|
* @return $this
|
||||||
|
*/
|
||||||
|
public function setParam(string $key, $value): self
|
||||||
|
{
|
||||||
|
$this->params[$key] = $value;
|
||||||
|
|
||||||
|
return $this;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param string $key
|
||||||
|
*
|
||||||
|
* @return mixed|null
|
||||||
|
*/
|
||||||
|
public function getParam(string $key)
|
||||||
|
{
|
||||||
|
return (isset($this->params[$key])) ? $this->params[$key] : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param string $namespace
|
||||||
|
*
|
||||||
|
* @return $this
|
||||||
|
*/
|
||||||
|
public function setNamespace(string $namespace): self
|
||||||
|
{
|
||||||
|
$this->namespace = $namespace;
|
||||||
|
|
||||||
|
return $this;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @return string
|
||||||
|
*/
|
||||||
|
public function getNamespace()
|
||||||
|
{
|
||||||
|
return $this->namespace;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Submit data to StatsD.
|
||||||
|
*/
|
||||||
|
public function submit(): void
|
||||||
|
{
|
||||||
|
$projectId = $this->params['projectId'] ?? '';
|
||||||
|
|
||||||
|
$storage = $this->params['storage'] ?? 0;
|
||||||
|
|
||||||
|
$networkRequestSize = $this->params['networkRequestSize'] ?? 0;
|
||||||
|
$networkResponseSize = $this->params['networkResponseSize'] ?? 0;
|
||||||
|
|
||||||
|
$httpMethod = $this->params['httpMethod'] ?? '';
|
||||||
|
$httpRequest = $this->params['httpRequest'] ?? 0;
|
||||||
|
|
||||||
|
$functionId = $this->params['functionId'] ?? '';
|
||||||
|
$functionExecution = $this->params['functionExecution'] ?? 0;
|
||||||
|
$functionExecutionTime = $this->params['functionExecutionTime'] ?? 0;
|
||||||
|
$functionStatus = $this->params['functionStatus'] ?? '';
|
||||||
|
|
||||||
|
$tags = ",project={$projectId},version=" . App::getEnv('_APP_VERSION', 'UNKNOWN');
|
||||||
|
|
||||||
|
// the global namespace is prepended to every key (optional)
|
||||||
|
$this->statsd->setNamespace($this->namespace);
|
||||||
|
|
||||||
|
if ($httpRequest >= 1) {
|
||||||
|
$this->statsd->increment('requests.all' . $tags . ',method=' . \strtolower($httpMethod));
|
||||||
|
}
|
||||||
|
|
||||||
|
if ($functionExecution >= 1) {
|
||||||
|
$this->statsd->increment('executions.all' . $tags . ',functionId=' . $functionId . ',functionStatus=' . $functionStatus);
|
||||||
|
$this->statsd->count('executions.time' . $tags . ',functionId=' . $functionId, $functionExecutionTime);
|
||||||
|
}
|
||||||
|
|
||||||
|
$this->statsd->count('network.inbound' . $tags, $networkRequestSize);
|
||||||
|
$this->statsd->count('network.outbound' . $tags, $networkResponseSize);
|
||||||
|
$this->statsd->count('network.all' . $tags, $networkRequestSize + $networkResponseSize);
|
||||||
|
|
||||||
|
if ($storage >= 1) {
|
||||||
|
$this->statsd->count('storage.all' . $tags, $storage);
|
||||||
|
}
|
||||||
|
|
||||||
|
$this->reset();
|
||||||
|
}
|
||||||
|
|
||||||
|
public function reset(): self
|
||||||
|
{
|
||||||
|
$this->params = [];
|
||||||
|
$this->namespace = 'appwrite.usage';
|
||||||
|
|
||||||
|
return $this;
|
||||||
|
}
|
||||||
|
}
|
69
tests/unit/Stats/StatsTest.php
Normal file
69
tests/unit/Stats/StatsTest.php
Normal file
|
@ -0,0 +1,69 @@
|
||||||
|
<?php
|
||||||
|
|
||||||
|
namespace Appwrite\Tests;
|
||||||
|
|
||||||
|
use Appwrite\Stats\Stats;
|
||||||
|
use PHPUnit\Framework\TestCase;
|
||||||
|
use Utopia\App;
|
||||||
|
|
||||||
|
class StatsTest extends TestCase
|
||||||
|
{
|
||||||
|
/**
|
||||||
|
* @var Stats
|
||||||
|
*/
|
||||||
|
protected $object = null;
|
||||||
|
|
||||||
|
public function setUp(): void
|
||||||
|
{
|
||||||
|
$host = App::getEnv('_APP_STATSD_HOST', 'telegraf');
|
||||||
|
$port = App::getEnv('_APP_STATSD_PORT', 8125);
|
||||||
|
|
||||||
|
$connection = new \Domnikl\Statsd\Connection\UdpSocket($host, $port);
|
||||||
|
$statsd = new \Domnikl\Statsd\Client($connection);
|
||||||
|
|
||||||
|
$this->object = new Stats($statsd);
|
||||||
|
}
|
||||||
|
|
||||||
|
public function tearDown(): void
|
||||||
|
{
|
||||||
|
}
|
||||||
|
|
||||||
|
public function testNamespace()
|
||||||
|
{
|
||||||
|
$this->object->setNamespace('appwritetest.usage');
|
||||||
|
$this->assertEquals('appwritetest.usage', $this->object->getNamespace());
|
||||||
|
}
|
||||||
|
|
||||||
|
public function testParams()
|
||||||
|
{
|
||||||
|
$this->object
|
||||||
|
->setParam('projectId', 'appwrite_test')
|
||||||
|
->setParam('networkRequestSize', 100)
|
||||||
|
;
|
||||||
|
|
||||||
|
$this->assertEquals('appwrite_test', $this->object->getParam('projectId'));
|
||||||
|
$this->assertEquals(100, $this->object->getParam('networkRequestSize'));
|
||||||
|
|
||||||
|
$this->object->submit();
|
||||||
|
|
||||||
|
$this->assertEquals(null, $this->object->getParam('projectId'));
|
||||||
|
$this->assertEquals(null, $this->object->getParam('networkRequestSize'));
|
||||||
|
}
|
||||||
|
|
||||||
|
public function testReset()
|
||||||
|
{
|
||||||
|
$this->object
|
||||||
|
->setParam('projectId', 'appwrite_test')
|
||||||
|
->setParam('networkRequestSize', 100)
|
||||||
|
;
|
||||||
|
|
||||||
|
$this->assertEquals('appwrite_test', $this->object->getParam('projectId'));
|
||||||
|
$this->assertEquals(100, $this->object->getParam('networkRequestSize'));
|
||||||
|
|
||||||
|
$this->object->reset();
|
||||||
|
|
||||||
|
$this->assertEquals(null, $this->object->getParam('projectId'));
|
||||||
|
$this->assertEquals(null, $this->object->getParam('networkRequestSize'));
|
||||||
|
$this->assertEquals('appwrite.usage', $this->object->getNamespace());
|
||||||
|
}
|
||||||
|
}
|
Loading…
Reference in a new issue