updates

app/controllers/shared/api.php

@@ -68,7 +68,7 @@ $databaseListener = function (string $event, Document $document, Document $proje
         case $document->getCollection() === 'databases':
             $queueForUsage->addMetric("{$project->getId()}", "databases", $value); // per project
             break;
-        case str_starts_with($document->getCollection(), 'database'): // collections
+        case str_starts_with($document->getCollection(), 'database_'): // collections
             $queueForUsage->addMetric("{$project->getId()}.{$document['databaseId']}", "collections", $value); // per database
             $queueForUsage->addMetric("{$project->getId()}", "collections", $value); // per project
             break;
@@ -80,7 +80,7 @@ $databaseListener = function (string $event, Document $document, Document $proje
         case $document->getCollection() === 'buckets':
             $queueForUsage->addMetric("{$project->getId()}", "buckets", $value); // per project
             break;
-        case $document->getCollection() === 'files':
+        case str_starts_with($document->getCollection(), 'bucket_'): // files
             $queueForUsage->addMetric("{$project->getId()}.{$document['bucketId']}", "files", $value); // per bucket
             $queueForUsage->addMetric("{$project->getId()}.{$document['bucketId']}", "files.storage", $document->getAttribute('sizeOriginal') * $value); // per bucket
             $queueForUsage->addMetric("{$project->getId()}", "files", $value); // per project
@@ -474,16 +474,19 @@ App::shutdown()
             }
         }
 
-        if ($project->getId() && !empty($route->getLabel('sdk.namespace', null))) {
+        if ($project->getId() && $project->getId() !== 'console') {
             $fileSize = 0;
             $file = $request->getFiles('file');
+
             if (!empty($file)) {
                 $fileSize = (\is_array($file['size']) && isset($file['size'][0])) ? $file['size'][0] : $file['size'];
             }
 
-            $queueForUsage->addMetric("{$project->getId()}", "network.inbound", $request->getSize() + $fileSize);
-            $queueForUsage->addMetric("{$project->getId()}", "network.outbound", $response->getSize());
-            $queueForUsage->trigger();
+            $queueForUsage
+                ->setProject($project)
+                ->addMetric("{$project->getId()}", "network.inbound", $request->getSize() + $fileSize)
+                ->addMetric("{$project->getId()}", "network.outbound", $response->getSize())
+                ->trigger();
 
             // $usage
             //     ->setParam('project.{scope}.network.inbound', $request->getSize() + $fileSize)

app/workers/usage.php

@@ -2,36 +2,114 @@
 
 require_once __DIR__ . '/../worker.php';
 
+use Swoole\Table;
 use Swoole\Timer;
+use Utopia\Database\Database;
+use Utopia\Database\Document;
+use Utopia\Database\Validator\Authorization;
 use Utopia\Queue\Message;
+use Utopia\CLI\Console;
 
-$stack = [];
+$stats = [];
+
+$periods['1h'] = 'Y-m-d H:00';
+$periods['1d'] =  'Y-m-d 00:00';
+$periods['inf'] =  'Y-m-d 00:00';
+
+//$stats = new Table(10000, 1);
+//$stats->column('namespace', Table::TYPE_STRING, 64);
+//$stats->column('key', Table::TYPE_STRING, 64);
+//$stats->column('value', Table::TYPE_INT);
+//$stats->create();
 
 $server->job()
     ->inject('message')
-    ->action(function (Message $message) use (&$stack) {
+    ->action(function (Message $message) use (&$stats) {
         $payload = $message->getPayload() ?? [];
+        $project = new Document($payload['project'] ?? []);
+
         foreach ($payload['metrics'] ?? [] as $metric) {
-            if (!isset($stack[$metric['namespace']][$metric['key']])) {
-                $stack[$metric['namespace']][$metric['key']] = $metric['value'];
+            $uniq = md5($metric['namespace'] . $metric['key']);
+
+//            if ($stats->exists($uniq)) {
+//                $stats->incr($uniq, 'value', $metric['value']);
+//                continue;
+//            }
+//
+//            $stats->set($uniq, [
+//                'projectInternalId' => $project->getInternalId(),
+//                'database' => $project->getAttribute('database'),
+//                'key' => $metric['key'],
+//                'value' => $metric['value'],
+//                ]);
+//
+
+            if (!isset($stats[$uniq])) {
+                $stats[$uniq] = [
+                    'projectInternalId' => $project->getInternalId(),
+                    'database' => $project->getAttribute('database'),
+                    'key' => $metric['namespace'] . '.' . $metric['key'],
+                    'value' => $metric['value']
+                ];
+
                 continue;
             }
-            $stack[$metric['namespace']][$metric['key']] += $metric['value'];
+
+                $stats[$uniq]['value'] += $metric['value'];
         }
     });
 
 $server
     ->workerStart()
-    ->action(function () use (&$stack) {
-        Timer::tick(30000, function () use (&$stack) {
-            $hourly = date('d-m-Y H:00', time());
-            $daily  = date('d-m-Y 00:00', time());
+    ->inject('register')
+    ->inject('cache')
+    ->inject('pools')
+    ->action(function ($register, $cache, $pools) use ($periods, &$stats) {
+                Timer::tick(30000, function () use ($register, $cache, $pools, $periods, &$stats) {
 
-            $chunk = array_slice($stack, 0, count($stack));
-            array_splice($stack, 0, count($stack));
-            var_dump($chunk);
+                    $slice = array_slice($stats, 0, count($stats));
+                    array_splice($stats, 0, count($stats));
 
-        });
+                    foreach ($slice as $metric) {
+                        foreach ($periods as $period => $format) {
+                             $time = date($format, time());
+                             $id = \md5("{$time}_{$period}_{$metric['key']}");
+
+                             $adapter = new Database(
+                                 $pools
+                                 ->get($metric['database'])
+                                 ->pop()
+                                 ->getResource(),
+                                 $cache
+                             );
+
+                             $adapter->setNamespace('_' . $metric['projectInternalId']);
+
+                            try {
+                                $document = Authorization::skip(fn() =>$adapter->getDocument('stats', $id));
+                                if ($document->isEmpty()) {
+                                    console::log("{$period}, {$time}, {$metric['key']}={$metric['value']}");
+                                    Authorization::skip(fn() => $adapter->createDocument('stats', new Document([
+                                        '$id' => $id,
+                                        'period' => $period,
+                                        'time' => $time,
+                                        'metric' => $metric['key'],
+                                        'value' => $metric['value'],
+                                        'type' => 0,
+                                        'region' => 'default',
+                                    ])));
+                                } else {
+                                    $value = $document->getAttribute('value') + $metric['value'];
+                                    console::info("{$document->getAttribute('period')}, {$document->getAttribute('time')}, {$document->getAttribute('metric')} = {$value}");
+                                    Authorization::skip(fn() => $adapter->updateDocument('stats', $document->getId(), $document->setAttribute('value', $value)));
+                                }
+                            } catch (\Exception $e) {
+                                console::error($e->getMessage());
+                            }
+                            $pools->reclaim();
+                        }
+                    }
+                });
     });
 
 $server->start();

src/Appwrite/Event/Usage.php

@@ -43,6 +43,7 @@ class Usage extends Event
         $client = new Client($this->queue, $this->connection);
 
         return $client->enqueue([
+            'project' => $this->getProject(),
             'metrics' => $this->metrics,
         ]);
     }