CodeMirror/appwrite
1
0
Fork 0
mirror of synced 2024-10-03 02:37:40 +13:00
Code Issues Projects Releases Wiki Activity

functions api usage

Browse source
This commit is contained in:
shimon 2022-12-07 19:56:38 +02:00
parent 5d63d2f199
commit 4c6b6d00e8
5 changed files with 127 additions and 160 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

32
app/controllers/api/functions.php
View file

@ -1361,8 +1361,7 @@ App::get('/v1/functions/:functionId/usage')
->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d']), 'Date range.', true)
->inject('response')
->inject('dbForProject')
->inject('project')
->action(function (string $functionId, string $range, Response $response, Database $dbForProject, Document $project) {
->action(function (string $functionId, string $range, Response $response, Database $dbForProject) {
$function = $dbForProject->getDocument('functions', $functionId);
@ -1374,12 +1373,12 @@ App::get('/v1/functions/:functionId/usage')
$stats = $usage = [];
$days = $periods[$range];
$metrics = [
$project->getId() . '.function.' . $function->getId() . '.deployments',
$project->getId() . '.function.' . $function->getId() . '.deployments.storage',
$project->getId() . '.' . $function->getId() . '.builds',
$project->getId() . '.' . $function->getId() . '.builds.storage',
$project->getId() . '.' . $function->getId() . '.executions',
$project->getId() . '.' . $function->getId() . '.executions.compute',
'function.' . $function->getId() . '.deployments',
'function.' . $function->getId() . '.deployments.storage',
$function->getId() . '.builds',
$function->getId() . '.builds.storage',
$function->getId() . '.executions',
$function->getId() . '.executions.compute',
];
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
@ -1444,20 +1443,19 @@ App::get('/v1/functions/usage')
->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d']), 'Date range.', true)
->inject('response')
->inject('dbForProject')
->inject('project')
->action(function (string $range, Response $response, Database $dbForProject, Document $project) {
->action(function (string $range, Response $response, Database $dbForProject) {
$periods = Config::getParam('usage', []);
$stats = $usage = [];
$days = $periods[$range];
$metrics = [
$project->getId() . '.functions',
$project->getId() . '.deployments',
$project->getId() . '.deployments.storage',
$project->getId() . '.builds',
$project->getId() . '.builds.storage',
$project->getId() . '.executions',
$project->getId() . '.executions.compute',
'functions',
'deployments',
'deployments.storage',
'builds',
'builds.storage',
'executions',
'executions.compute',
];
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {

160
app/controllers/api/storage.php
View file

@ -1413,6 +1413,81 @@ App::delete('/v1/storage/buckets/:bucketId/files/:fileId')
$response->noContent();
});
App::get('/v1/storage/:bucketId/usage')
->desc('Get usage stats for storage bucket')
->groups(['api', 'storage'])
->label('scope', 'files.read')
->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
->label('sdk.namespace', 'storage')
->label('sdk.method', 'getBucketUsage')
->label('sdk.response.code', Response::STATUS_CODE_OK)
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
->label('sdk.response.model', Response::MODEL_USAGE_BUCKETS)
->param('bucketId', '', new UID(), 'Bucket ID.')
->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true)
->inject('response')
->inject('dbForProject')
->action(function (string $bucketId, string $range, Response $response, Database $dbForProject) {
$bucket = $dbForProject->getDocument('buckets', $bucketId);
if ($bucket->isEmpty()) {
throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND);
}
$periods = Config::getParam('usage', []);
$stats = $usage = [];
$days = $periods[$range];
$metrics = [
$bucket->getId() . '.files',
$bucket->getId() . '.files.storage',
];
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
foreach ($metrics as $metric) {
$limit = $days['limit'];
$period = $days['period'];
$results = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric] = [];
foreach ($results as $result) {
$stats[$metric][$result->getAttribute('time')] = [
'value' => $result->getAttribute('value'),
];
}
}
});
$format = match ($days['period']) {
'1h' => 'Y-m-d\TH:00:00.000P',
'1d' => 'Y-m-d\T00:00:00.000P',
};
foreach ($metrics as $metric) {
$usage[$metric] = [];
$leap = time() - ($days['limit'] * $days['factor']);
while ($leap < time()) {
$leap += $days['factor'];
$formatDate = date($format, $leap);
$usage[$metric][] = [
'value' => $stats[$metric][$formatDate]['value'] ?? 0,
'date' => $formatDate,
];
}
$usage[$metric] = array_reverse($usage[$metric]);
}
$response->dynamic(new Document([
'range' => $range,
'filesCount' => $usage[$metrics[0]],
'filesStorage' => $usage[$metrics[1]],
]), Response::MODEL_USAGE_BUCKETS);
});
App::get('/v1/storage/usage')
->desc('Get usage stats for storage')
->groups(['api', 'storage'])
@ -1426,16 +1501,15 @@ App::get('/v1/storage/usage')
->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true)
->inject('response')
->inject('dbForProject')
->inject('project')
->action(function (string $range, Response $response, Database $dbForProject, Document $project) {
->action(function (string $range, Response $response, Database $dbForProject) {
$periods = Config::getParam('usage', []);
$stats = $usage = [];
$days = $periods[$range];
$metrics = [
$project->getId() . '.buckets',
$project->getId() . '.files',
$project->getId() . '.files.storage',
'buckets',
'files',
'files.storage',
];
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
@ -1483,79 +1557,3 @@ App::get('/v1/storage/usage')
'filesStorage' => $usage[$metrics[2]],
]), Response::MODEL_USAGE_STORAGE);
});
App::get('/v1/storage/:bucketId/usage')
->desc('Get usage stats for a storage bucket')
->groups(['api', 'storage'])
->label('scope', 'files.read')
->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
->label('sdk.namespace', 'storage')
->label('sdk.method', 'getBucketUsage')
->label('sdk.response.code', Response::STATUS_CODE_OK)
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
->label('sdk.response.model', Response::MODEL_USAGE_BUCKETS)
->param('bucketId', '', new UID(), 'Bucket ID.')
->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true)
->inject('response')
->inject('dbForProject')
->inject('project')
->action(function (string $bucketId, string $range, Response $response, Database $dbForProject, Document $project) {
$bucket = $dbForProject->getDocument('buckets', $bucketId);
if ($bucket->isEmpty()) {
throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND);
}
$periods = Config::getParam('usage', []);
$stats = $usage = [];
$days = $periods[$range];
$metrics = [
$project->getId() . '.' . $bucket->getId() . '.files',
$project->getId() . '.' . $bucket->getId() . '.files.storage',
];
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
foreach ($metrics as $metric) {
$limit = $days['limit'];
$period = $days['period'];
$results = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric] = [];
foreach ($results as $result) {
$stats[$metric][$result->getAttribute('time')] = [
'value' => $result->getAttribute('value'),
];
}
}
});
$format = match ($days['period']) {
'1h' => 'Y-m-d\TH:00:00.000P',
'1d' => 'Y-m-d\T00:00:00.000P',
};
foreach ($metrics as $metric) {
$usage[$metric] = [];
$leap = time() - ($days['limit'] * $days['factor']);
while ($leap < time()) {
$leap += $days['factor'];
$formatDate = date($format, $leap);
$usage[$metric][] = [
'value' => $stats[$metric][$formatDate]['value'] ?? 0,
'date' => $formatDate,
];
}
$usage[$metric] = array_reverse($usage[$metric]);
}
$response->dynamic(new Document([
'range' => $range,
'filesCount' => $usage[$metrics[0]],
'filesStorage' => $usage[$metrics[1]],
]), Response::MODEL_USAGE_BUCKETS);
});

66
app/controllers/shared/api.php
View file

@ -58,55 +58,55 @@ $databaseListener = function (string $event, Document $document, Document $proje
switch (true) {
case $document->getCollection() === 'users':
$queueForUsage->addMetric("{$project->getId()}", "users", $value); // per project
$queueForUsage->addMetric("users", $value); // per project
break;
case $document->getCollection() === 'teams':
$queueForUsage->addMetric("{$project->getId()}", "teams", $value); // per project
$queueForUsage->addMetric("teams", $value); // per project
break;
case $document->getCollection() === 'sessions':
$queueForUsage->addMetric("{$project->getId()}", "sessions", $value); // per project
$queueForUsage->addMetric("sessions", $value); // per project
break;
case $document->getCollection() === 'databases':
$queueForUsage->addMetric("{$project->getId()}", "databases", $value); // per project
$queueForUsage->addMetric("databases", $value); // per project
break;
case str_starts_with($document->getCollection(), 'database_'): // collections
$queueForUsage->addMetric("{$project->getId()}.{$document['databaseId']}", "collections", $value); // per database
$queueForUsage->addMetric("{$project->getId()}", "collections", $value); // per project
$queueForUsage->addMetric("{$document['databaseId']}" . ".collections", $value); // per database
$queueForUsage->addMetric("collections", $value); // per project
break;
case $document->getCollection() === 'documents':
$queueForUsage->addMetric("{$project->getId()}.{$document['databaseId']}.{$document['collectionId']}", "documents", $value); // per collection
$queueForUsage->addMetric("{$project->getId()}.{$document['databaseId']}", "documents", $value); // per database
$queueForUsage->addMetric("{$project->getId()}", "documents", $value); // per project
$queueForUsage->addMetric("{$document['databaseId']}" . "." . "{$document['collectionId']}" . ".documents", $value); // per collection
$queueForUsage->addMetric("{$document['databaseId']}" . ".documents", $value); // per database
$queueForUsage->addMetric("documents", $value); // per project
break;
case $document->getCollection() === 'buckets':
$queueForUsage->addMetric("{$project->getId()}", "buckets", $value); // per project
$queueForUsage->addMetric("buckets", $value); // per project
break;
case str_starts_with($document->getCollection(), 'bucket_'): // files
$queueForUsage->addMetric("{$project->getId()}.{$document['bucketId']}", "files", $value); // per bucket
$queueForUsage->addMetric("{$project->getId()}.{$document['bucketId']}", "files.storage", $document->getAttribute('sizeOriginal') * $value); // per bucket
$queueForUsage->addMetric("{$project->getId()}", "files", $value); // per project
$queueForUsage->addMetric("{$project->getId()}", "files.storage", $document->getAttribute('sizeOriginal') * $value); // per project
$queueForUsage->addMetric("{$document['bucketId']}" . ".files", $value); // per bucket
$queueForUsage->addMetric("{$document['bucketId']}" . ".files.storage", $document->getAttribute('sizeOriginal') * $value); // per bucket
$queueForUsage->addMetric("files", $value); // per project
$queueForUsage->addMetric("files.storage", $document->getAttribute('sizeOriginal') * $value); // per project
break;
case $document->getCollection() === 'functions':
$queueForUsage->addMetric("{$project->getId()}", "functions", $value); // per project
$queueForUsage->addMetric("functions", $value); // per project
break;
case $document->getCollection() === 'deployments':
$queueForUsage->addMetric("{$project->getId()}.{$document['resourceType']}.{$document['resourceId']}", "deployments", $value); // per function
$queueForUsage->addMetric("{$project->getId()}.{$document['resourceType']}.{$document['resourceId']}", "deployments.storage", $document->getAttribute('size') * $value); // per function
$queueForUsage->addMetric("{$project->getId()}", "deployments", $value); // per project
$queueForUsage->addMetric("{$project->getId()}", "deployments.storage", $document->getAttribute('size') * $value); // per project
$queueForUsage->addMetric("{$document['resourceType']}" . "." . "{$document['resourceId']}" . ".deployments", $value); // per function
$queueForUsage->addMetric("{$document['resourceType']}" . "." . "{$document['resourceId']}" . ".deployments.storage", $document->getAttribute('size') * $value); // per function
$queueForUsage->addMetric("deployments", $value); // per project
$queueForUsage->addMetric("deployments.storage", $document->getAttribute('size') * $value); // per project
break;
case $document->getCollection() === 'builds': // needs to extract functionId
$queueForUsage->addMetric("{$project->getId()}.{$document['functionId']}", "builds", $value); // per function
$queueForUsage->addMetric("{$project->getId()}.{$document['functionId']}", "builds.storage", $document->getAttribute('size') * $value); // per function
$queueForUsage->addMetric("{$project->getId()}", "builds", $value); // per project
$queueForUsage->addMetric("{$project->getId()}", "builds.storage", $document->getAttribute('size') * $value); // per project
$queueForUsage->addMetric("{$document['functionId']}" . ".builds", $value); // per function
$queueForUsage->addMetric("{$document['functionId']}" . ".builds.storage", $document->getAttribute('size') * $value); // per function
$queueForUsage->addMetric("builds", $value); // per project
$queueForUsage->addMetric("builds.storage", $document->getAttribute('size') * $value); // per project
break;
case $document->getCollection() === 'executions':
$queueForUsage->addMetric("{$project->getId()}.{$document['functionId']}", "executions", $value); // per function
$queueForUsage->addMetric("{$project->getId()}.{$document['functionId']}", "executions.compute", $document->getAttribute('duration') * $value); // per function
$queueForUsage->addMetric("{$project->getId()}", "executions", $value); // per project
$queueForUsage->addMetric("{$project->getId()}", "executions.compute", $document->getAttribute('duration') * $value); // per project
$queueForUsage->addMetric("{$document['functionId']}" . ".executions", $value); // per function
$queueForUsage->addMetric("{$document['functionId']}" . ".executions.compute", $document->getAttribute('duration') * $value); // per function
$queueForUsage->addMetric("executions", $value); // per project
$queueForUsage->addMetric("executions.compute", $document->getAttribute('duration') * $value); // per project
break;
default:
break;
@ -214,14 +214,6 @@ App::init()
->setProject($project)
->setUser($user);
// $usage
// ->setParam('projectInternalId', $project->getInternalId())
// ->setParam('projectId', $project->getId())
// ->setParam('project.{scope}.network.requests', 1)
// ->setParam('httpMethod', $request->getMethod())
// ->setParam('project.{scope}.network.inbound', 0)
// ->setParam('project.{scope}.network.outbound', 0);
$deletes->setProject($project);
$database->setProject($project);
@ -469,8 +461,8 @@ App::shutdown()
$queueForUsage
->setProject($project)
->addMetric("{$project->getId()}", "network.inbound", $request->getSize() + $fileSize)
->addMetric("{$project->getId()}", "network.outbound", $response->getSize())
->addMetric("network.inbound", $request->getSize() + $fileSize)
->addMetric("network.outbound", $response->getSize())
->trigger();
}
});

25
app/workers/usage.php
View file

@ -17,12 +17,6 @@ $periods['1h'] = 'Y-m-d H:00';
$periods['1d'] = 'Y-m-d 00:00';
$periods['inf'] = 'Y-m-d 00:00';
//$stats = new Table(10000, 1);
//$stats->column('namespace', Table::TYPE_STRING, 64);
//$stats->column('key', Table::TYPE_STRING, 64);
//$stats->column('value', Table::TYPE_INT);
//$stats->create();
$server->job()
->inject('message')
->action(function (Message $message) use (&$stats) {
@ -30,33 +24,20 @@ $server->job()
$project = new Document($payload['project'] ?? []);
foreach ($payload['metrics'] ?? [] as $metric) {
$uniq = md5($metric['namespace'] . $metric['key']);
// if ($stats->exists($uniq)) {
// $stats->incr($uniq, 'value', $metric['value']);
// continue;
// }
//
// $stats->set($uniq, [
// 'projectInternalId' => $project->getInternalId(),
// 'database' => $project->getAttribute('database'),
// 'key' => $metric['key'],
// 'value' => $metric['value'],
// ]);
//
$uniq = md5($metric['key']);
if (!isset($stats[$uniq])) {
$stats[$uniq] = [
'projectInternalId' => $project->getInternalId(),
'database' => $project->getAttribute('database'),
'key' => $metric['namespace'] . '.' . $metric['key'],
'key' => $metric['key'],
'value' => $metric['value']
];
continue;
}
$stats[$uniq]['value'] += $metric['value'];
$stats[$uniq]['value'] += $metric['value'];
}
});

4
src/Appwrite/Event/Usage.php
View file

@ -17,15 +17,13 @@ class Usage extends Event
/**
* Add metric.
*
* @param string $namespace
* @param string $key
* @param int $value
* @return self
*/
public function addMetric(string $namespace, string $key, int $value): self
public function addMetric(string $key, int $value): self
{
$this->metrics[] = [
'namespace' => $namespace,
'key' => $key,
'value' => $value,
];