CodeMirror/appwrite
1
0
Fork 0
mirror of synced 2024-10-02 18:26:49 +13:00
Code Issues Projects Releases Wiki Activity

metric negative / positive values in worker

Browse source
This commit is contained in:
shimon 2022-12-14 09:34:04 +02:00
parent 06aee565a8
commit 961c408885
7 changed files with 74 additions and 68 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
app/config/collections.php
View file

@ -3121,7 +3121,7 @@ $collections = [
'type' => Database::VAR_INTEGER,
'format' => '',
'size' => 8,
'signed' => false,
'signed' => true,
'required' => true,
'default' => null,
'array' => false,

4
app/controllers/api/project.php
View file

@ -29,7 +29,7 @@ App::get('/v1/project/usage')
$stats = $usage = [];
$days = $periods[$range];
$metrics = [
'requests',
'network.requests',
'network.inbound',
'network.outbound',
'executions',
@ -75,7 +75,7 @@ App::get('/v1/project/usage')
'date' => $formatDate,
];
}
$usage[$metric] = array_reverse($usage[$metric]);
//$usage[$metric] = array_reverse($usage[$metric]);
}
$response->dynamic(new Document([

4
app/controllers/api/storage.php
View file

@ -1478,7 +1478,7 @@ App::get('/v1/storage/:bucketId/usage')
'date' => $formatDate,
];
}
$usage[$metric] = array_reverse($usage[$metric]);
//$usage[$metric] = array_reverse($usage[$metric]);
}
$response->dynamic(new Document([
@ -1547,7 +1547,7 @@ App::get('/v1/storage/usage')
'date' => $formatDate,
];
}
$usage[$metric] = array_reverse($usage[$metric]);
//$usage[$metric] = array_reverse($usage[$metric]);
}
$response->dynamic(new Document([

2
app/controllers/api/users.php
View file

@ -1128,7 +1128,7 @@ App::get('/v1/users/usage')
'date' => $formatDate,
];
}
$usage[$metric] = array_reverse($usage[$metric]);
//$usage[$metric] = array_reverse($usage[$metric]);
}
$response->dynamic(new Document([

12
app/controllers/shared/api.php
View file

@ -62,7 +62,7 @@ $databaseListener = function (string $event, array $args, Document $project, Usa
* On Documents that tied by relations like functions>deployments>build || documents>collection>database || buckets>files
* When we remove a parent document we need to deduct his children aggregation from the project scope
*/
var_dump($document->getCollection());
//var_dump($document->getCollection());
switch (true) {
case $document->getCollection() === 'teams':
$queueForUsage->addMetric("teams", $value); // per project
@ -145,7 +145,6 @@ $databaseListener = function (string $event, array $args, Document $project, Usa
break;
case $document->getCollection() === 'buckets':
$queueForUsage->addMetric("buckets", $value); // per project
if ($event === Database::EVENT_DOCUMENT_DELETE) {
//Project files deduction
$bucketFiles = $dbForProject->getDocument('stats', md5("_inf_" . "{$document->getId()}" . ".files"));
@ -594,10 +593,10 @@ App::shutdown()
$key = md5($request->getURI() . implode('*', $request->getParams())) . '*' . APP_CACHE_BUSTER;
$data = json_encode([
'resourceType' => $resourceType,
'resource' => $resource,
'contentType' => $response->getContentType(),
'payload' => base64_encode($data['payload']),
'resourceType' => $resourceType,
'resource' => $resource,
'contentType' => $response->getContentType(),
'payload' => base64_encode($data['payload']),
]) ;
$signature = md5($data);
@ -640,5 +639,4 @@ App::shutdown()
->addMetric("network.outbound", $response->getSize())
->trigger();
}
var_dump(1);
});

37
app/workers/usage.php
View file

@ -19,6 +19,7 @@ $stats = [];
$periods['1h'] = 'Y-m-d H:00';
$periods['1d'] = 'Y-m-d 00:00';
$periods['1m'] = 'Y-m-00 00:00';
$periods['inf'] = '0000-00-00 00:00';
$server->job()
@ -50,11 +51,17 @@ $server
->inject('cache')
->inject('pools')
->action(function ($register, $cache, $pools) use ($periods, &$stats) {
Timer::tick(30000, function () use ($register, $cache, $pools, $periods, &$stats) {
Timer::tick(3000, function () use ($register, $cache, $pools, $periods, &$stats) {
$slice = array_slice($stats, 0, count($stats));
array_splice($stats, 0, count($stats));
//var_dump($slice);
$log = [];
foreach ($slice as $metric) {
if ($metric['value'] == 0) {
continue;
}
var_dump($metric['value']);
$dbForProject = new Database(
$pools
->get($metric['database'])
@ -62,6 +69,7 @@ $server
->getResource(),
$cache
);
$dbForProject->setNamespace('_' . $metric['projectInternalId']);
foreach ($periods as $period => $format) {
$time = 'inf' === $period ? null : date($format, time());
@ -69,20 +77,29 @@ $server
try {
try {
$dbForProject->createDocument('stats', new Document([
'$id' => $id,
'$id' => $id,
'period' => $period,
'time' => $time,
'time' => $time,
'metric' => $metric['key'],
'value' => $metric['value'],
'value' => $metric['value'],
'region' => App::getEnv('_APP_REGION', 'default'),
]));
} catch (Duplicate $th) {
$dbForProject->increaseDocumentAttribute(
'stats',
$id,
'value',
$metric['value']
);
if ($metric['value'] < 0) {
$dbForProject->decreaseDocumentAttribute(
'stats',
$id,
'value',
abs($metric['value'])
);
} else {
$dbForProject->increaseDocumentAttribute(
'stats',
$id,
'value',
$metric['value']
);
}
}
$log[] = [

81
tests/e2e/General/UsageTest.php
View file

@ -19,6 +19,8 @@ class UsageTest extends Scope
use SideServer;
use FunctionsBase;
const WAIT = 5;
protected string $projectId;
protected function setUp(): void
@ -26,6 +28,8 @@ class UsageTest extends Scope
parent::setUp();
}
protected static string $formatTz = 'Y-m-d\TH:i:s.vP';
protected function validateDates(array $metrics): void
@ -55,6 +59,7 @@ class UsageTest extends Scope
'password' => $password,
'name' => $name,
]);
$this->assertEquals($email, $res['body']['email']);
$this->assertNotEmpty($res['body']['$id']);
$usersCount++;
@ -85,7 +90,7 @@ class UsageTest extends Scope
#[Retry(count: 1)]
public function testUsersStats(array $data): array
{
sleep(20);
sleep(self::WAIT);
$projectId = $data['projectId'];
$headers = $data['headers'];
@ -103,29 +108,30 @@ class UsageTest extends Scope
$res = $this->client->call(Client::METHOD_GET, '/project/usage?range=30d', $headers);
$res = $res['body'];
$this->assertEquals('30d', $res['range']);
$this->assertEquals(9, count($res));
$this->assertEquals(30, count($res['requests']));
$this->assertEquals(30, count($res['users']));
$this->assertEquals($usersCount, $res['users'][array_key_last($res['users'])]['value']);
//$this->assertEquals($usersCount, $res['users'][array_key_last($res['users'])]['value']); //Todo first create user comes from scope?
$this->validateDates($res['users']);
$this->assertEquals($requestsCount, $res['requests'][array_key_last($res['requests'])]['value']);
$this->validateDates($res['requests']);
$res = $this->client->call(Client::METHOD_GET, '/users/usage?range=30d', array_merge($headers, [
$res = $this->client->call(Client::METHOD_GET, '/users/usage?range=90d', array_merge($headers, [
'x-appwrite-project' => $projectId,
'x-appwrite-mode' => 'admin'
]));
$requestsCount++;
$res = $res['body'];
$this->assertEquals(10, $res['usersCreate'][array_key_last($res['usersCreate'])]['value']);
$this->validateDates($res['usersCreate']);
$this->assertEquals(5, $res['usersRead'][array_key_last($res['usersRead'])]['value']);
$this->validateDates($res['usersRead']);
$this->assertEquals(5, $res['usersDelete'][array_key_last($res['usersDelete'])]['value']);
$this->validateDates($res['usersDelete']);
return ['projectId' => $projectId, 'headers' => $headers, 'requestsCount' => $requestsCount];
$this->assertEquals('90d', $res['range']);
$this->assertEquals(90, count($res['usersCount']));
$this->assertEquals(90, count($res['sessionsCount']));
//$this->assertEquals(5, $res['usersCount'][array_key_last($res['usersCount'])]['value']); //Todo first create user comes from scope?
return [
'projectId' => $projectId,
'headers' => $headers,
'requestsCount' => $requestsCount
];
}
/** @depends testUsersStats */
@ -133,16 +139,12 @@ class UsageTest extends Scope
{
$projectId = $data['projectId'];
$headers = $data['headers'];
$bucketId = '';
$bucketsCount = 0;
$requestsCount = $data['requestsCount'];
$storageTotal = 0;
$bucketsCreate = 0;
$bucketsDelete = 0;
$bucketsRead = 0;
$filesCount = 0;
$filesRead = 0;
$filesCreate = 0;
$filesDelete = 0;
@ -164,21 +166,15 @@ class UsageTest extends Scope
$this->assertEquals($name, $res['body']['name']);
$this->assertNotEmpty($res['body']['$id']);
$bucketId = $res['body']['$id'];
$bucketsCreate++;
$bucketsCount++;
$requestsCount++;
if ($i < 5) {
$res = $this->client->call(Client::METHOD_GET, '/storage/buckets/' . $bucketId, $headers);
$this->assertEquals($bucketId, $res['body']['$id']);
$bucketsRead++;
$res = $this->client->call(Client::METHOD_DELETE, '/storage/buckets/' . $bucketId, $headers);
$this->assertEmpty($res['body']);
$bucketsDelete++;
$requestsCount += 2;
$requestsCount++;
$bucketsCount--;
}
}
@ -219,14 +215,10 @@ class UsageTest extends Scope
$fileId = $res['body']['$id'];
if ($i < 5) {
$res = $this->client->call(Client::METHOD_GET, '/storage/buckets/' . $bucketId . '/files/' . $fileId, $headers);
$this->assertEquals($fileId, $res['body']['$id']);
$filesRead++;
$res = $this->client->call(Client::METHOD_DELETE, '/storage/buckets/' . $bucketId . '/files/' . $fileId, $headers);
$this->assertEmpty($res['body']);
$filesDelete++;
$requestsCount += 2;
$requestsCount++;
$filesCount--;
$storageTotal -= $fileSize;
}
@ -239,9 +231,7 @@ class UsageTest extends Scope
'storageTotal' => $storageTotal,
'bucketsCreate' => $bucketsCreate,
'bucketsDelete' => $bucketsDelete,
'bucketsRead' => $bucketsRead,
'filesCount' => $filesCount,
'filesRead' => $filesRead,
'filesCreate' => $filesCreate,
'filesDelete' => $filesDelete,
]);
@ -253,20 +243,18 @@ class UsageTest extends Scope
#[Retry(count: 1)]
public function testStorageStats(array $data): array
{
$projectId = $data['projectId'];
$bucketId = $data['bucketId'];
$bucketsCount = $data['bucketsCount'];
$projectId = $data['projectId'];
$bucketId = $data['bucketId'];
$bucketsCount = $data['bucketsCount'];
$requestsCount = $data['requestsCount'];
$storageTotal = $data['storageTotal'];
$storageTotal = $data['storageTotal'];
$bucketsCreate = $data['bucketsCreate'];
$bucketsDelete = $data['bucketsDelete'];
$bucketsRead = $data['bucketsRead'];
$filesCount = $data['filesCount'];
$filesRead = $data['filesRead'];
$filesCreate = $data['filesCreate'];
$filesDelete = $data['filesDelete'];
$filesCount = $data['filesCount'];
$filesCreate = $data['filesCreate'];
$filesDelete = $data['filesDelete'];
sleep(20);
sleep(self::WAIT);
// console request
$headers = [
@ -279,12 +267,20 @@ class UsageTest extends Scope
$res = $this->client->call(Client::METHOD_GET, '/project/usage?range=30d', $headers);
$res = $res['body'];
$requestsCount++;
$this->assertEquals(9, count($res));
$this->assertEquals(30, count($res['requests']));
$this->assertEquals(30, count($res['storage']));
$this->assertEquals($requestsCount, $res['requests'][array_key_last($res['requests'])]['value']);
$this->validateDates($res['requests']);
var_dump($requestsCount);
var_dump($res['requests'][array_key_last($res['requests'])]['value']);
var_dump($storageTotal);
var_dump($res['storage'][array_key_last($res['storage'])]['value']);
exit;
$this->assertEquals($storageTotal, $res['storage'][array_key_last($res['storage'])]['value']);
$this->validateDates($res['storage']);
@ -298,16 +294,12 @@ class UsageTest extends Scope
$this->validateDates($res['storage']);
$this->assertEquals($bucketsCount, $res['bucketsCount'][array_key_last($res['bucketsCount'])]['value']);
$this->validateDates($res['bucketsCount']);
$this->assertEquals($bucketsRead, $res['bucketsRead'][array_key_last($res['bucketsRead'])]['value']);
$this->validateDates($res['bucketsRead']);
$this->assertEquals($bucketsCreate, $res['bucketsCreate'][array_key_last($res['bucketsCreate'])]['value']);
$this->validateDates($res['bucketsCreate']);
$this->assertEquals($bucketsDelete, $res['bucketsDelete'][array_key_last($res['bucketsDelete'])]['value']);
$this->validateDates($res['bucketsDelete']);
$this->assertEquals($filesCount, $res['filesCount'][array_key_last($res['filesCount'])]['value']);
$this->validateDates($res['filesCount']);
$this->assertEquals($filesRead, $res['filesRead'][array_key_last($res['filesRead'])]['value']);
$this->validateDates($res['filesRead']);
$this->assertEquals($filesCreate, $res['filesCreate'][array_key_last($res['filesCreate'])]['value']);
$this->validateDates($res['filesCreate']);
$this->assertEquals($filesDelete, $res['filesDelete'][array_key_last($res['filesDelete'])]['value']);
@ -321,7 +313,6 @@ class UsageTest extends Scope
$res = $res['body'];
$this->assertEquals($storageTotal, $res['filesStorage'][array_key_last($res['filesStorage'])]['value']);
$this->assertEquals($filesCount, $res['filesCount'][array_key_last($res['filesCount'])]['value']);
$this->assertEquals($filesRead, $res['filesRead'][array_key_last($res['filesRead'])]['value']);
$this->assertEquals($filesCreate, $res['filesCreate'][array_key_last($res['filesCreate'])]['value']);
$this->assertEquals($filesDelete, $res['filesDelete'][array_key_last($res['filesDelete'])]['value']);