From f44ee4ec906833fc065e8519a3f2bdfe3be43ca9 Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 6 Dec 2022 13:36:17 +0200 Subject: [PATCH] updates --- app/controllers/shared/api.php | 15 +++-- app/workers/usage.php | 104 ++++++++++++++++++++++++++++----- src/Appwrite/Event/Usage.php | 1 + 3 files changed, 101 insertions(+), 19 deletions(-) diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 0d16a4641c..34eeca9d87 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -68,7 +68,7 @@ $databaseListener = function (string $event, Document $document, Document $proje case $document->getCollection() === 'databases': $queueForUsage->addMetric("{$project->getId()}", "databases", $value); // per project break; - case str_starts_with($document->getCollection(), 'database'): // collections + case str_starts_with($document->getCollection(), 'database_'): // collections $queueForUsage->addMetric("{$project->getId()}.{$document['databaseId']}", "collections", $value); // per database $queueForUsage->addMetric("{$project->getId()}", "collections", $value); // per project break; @@ -80,7 +80,7 @@ $databaseListener = function (string $event, Document $document, Document $proje case $document->getCollection() === 'buckets': $queueForUsage->addMetric("{$project->getId()}", "buckets", $value); // per project break; - case $document->getCollection() === 'files': + case str_starts_with($document->getCollection(), 'bucket_'): // files $queueForUsage->addMetric("{$project->getId()}.{$document['bucketId']}", "files", $value); // per bucket $queueForUsage->addMetric("{$project->getId()}.{$document['bucketId']}", "files.storage", $document->getAttribute('sizeOriginal') * $value); // per bucket $queueForUsage->addMetric("{$project->getId()}", "files", $value); // per project @@ -474,16 +474,19 @@ App::shutdown() } } - if ($project->getId() && !empty($route->getLabel('sdk.namespace', null))) { + if ($project->getId() && $project->getId() !== 'console') { $fileSize = 0; $file = $request->getFiles('file'); + if (!empty($file)) { $fileSize = (\is_array($file['size']) && isset($file['size'][0])) ? $file['size'][0] : $file['size']; } - $queueForUsage->addMetric("{$project->getId()}", "network.inbound", $request->getSize() + $fileSize); - $queueForUsage->addMetric("{$project->getId()}", "network.outbound", $response->getSize()); - $queueForUsage->trigger(); + $queueForUsage + ->setProject($project) + ->addMetric("{$project->getId()}", "network.inbound", $request->getSize() + $fileSize) + ->addMetric("{$project->getId()}", "network.outbound", $response->getSize()) + ->trigger(); // $usage // ->setParam('project.{scope}.network.inbound', $request->getSize() + $fileSize) diff --git a/app/workers/usage.php b/app/workers/usage.php index fda09de1b9..1c1e4d2dcd 100644 --- a/app/workers/usage.php +++ b/app/workers/usage.php @@ -2,36 +2,114 @@ require_once __DIR__ . '/../worker.php'; +use Swoole\Table; use Swoole\Timer; +use Utopia\Database\Database; +use Utopia\Database\Document; +use Utopia\Database\Validator\Authorization; use Utopia\Queue\Message; +use Utopia\CLI\Console; -$stack = []; +$stats = []; + +$periods['1h'] = 'Y-m-d H:00'; +$periods['1d'] = 'Y-m-d 00:00'; +$periods['inf'] = 'Y-m-d 00:00'; + +//$stats = new Table(10000, 1); +//$stats->column('namespace', Table::TYPE_STRING, 64); +//$stats->column('key', Table::TYPE_STRING, 64); +//$stats->column('value', Table::TYPE_INT); +//$stats->create(); $server->job() ->inject('message') - ->action(function (Message $message) use (&$stack) { + ->action(function (Message $message) use (&$stats) { $payload = $message->getPayload() ?? []; + $project = new Document($payload['project'] ?? []); + foreach ($payload['metrics'] ?? [] as $metric) { - if (!isset($stack[$metric['namespace']][$metric['key']])) { - $stack[$metric['namespace']][$metric['key']] = $metric['value']; + $uniq = md5($metric['namespace'] . $metric['key']); + +// if ($stats->exists($uniq)) { +// $stats->incr($uniq, 'value', $metric['value']); +// continue; +// } +// +// $stats->set($uniq, [ +// 'projectInternalId' => $project->getInternalId(), +// 'database' => $project->getAttribute('database'), +// 'key' => $metric['key'], +// 'value' => $metric['value'], +// ]); +// + + if (!isset($stats[$uniq])) { + $stats[$uniq] = [ + 'projectInternalId' => $project->getInternalId(), + 'database' => $project->getAttribute('database'), + 'key' => $metric['namespace'] . '.' . $metric['key'], + 'value' => $metric['value'] + ]; + continue; } - $stack[$metric['namespace']][$metric['key']] += $metric['value']; + + $stats[$uniq]['value'] += $metric['value']; } }); $server ->workerStart() - ->action(function () use (&$stack) { - Timer::tick(30000, function () use (&$stack) { - $hourly = date('d-m-Y H:00', time()); - $daily = date('d-m-Y 00:00', time()); + ->inject('register') + ->inject('cache') + ->inject('pools') + ->action(function ($register, $cache, $pools) use ($periods, &$stats) { + Timer::tick(30000, function () use ($register, $cache, $pools, $periods, &$stats) { - $chunk = array_slice($stack, 0, count($stack)); - array_splice($stack, 0, count($stack)); - var_dump($chunk); + $slice = array_slice($stats, 0, count($stats)); + array_splice($stats, 0, count($stats)); - }); + foreach ($slice as $metric) { + foreach ($periods as $period => $format) { + $time = date($format, time()); + $id = \md5("{$time}_{$period}_{$metric['key']}"); + + $adapter = new Database( + $pools + ->get($metric['database']) + ->pop() + ->getResource(), + $cache + ); + + $adapter->setNamespace('_' . $metric['projectInternalId']); + + try { + $document = Authorization::skip(fn() =>$adapter->getDocument('stats', $id)); + if ($document->isEmpty()) { + console::log("{$period}, {$time}, {$metric['key']}={$metric['value']}"); + Authorization::skip(fn() => $adapter->createDocument('stats', new Document([ + '$id' => $id, + 'period' => $period, + 'time' => $time, + 'metric' => $metric['key'], + 'value' => $metric['value'], + 'type' => 0, + 'region' => 'default', + ]))); + } else { + $value = $document->getAttribute('value') + $metric['value']; + console::info("{$document->getAttribute('period')}, {$document->getAttribute('time')}, {$document->getAttribute('metric')} = {$value}"); + Authorization::skip(fn() => $adapter->updateDocument('stats', $document->getId(), $document->setAttribute('value', $value))); + } + } catch (\Exception $e) { + console::error($e->getMessage()); + } + $pools->reclaim(); + } + } + }); }); $server->start(); diff --git a/src/Appwrite/Event/Usage.php b/src/Appwrite/Event/Usage.php index ab5cae15d8..1da63383d8 100644 --- a/src/Appwrite/Event/Usage.php +++ b/src/Appwrite/Event/Usage.php @@ -43,6 +43,7 @@ class Usage extends Event $client = new Client($this->queue, $this->connection); return $client->enqueue([ + 'project' => $this->getProject(), 'metrics' => $this->metrics, ]); }