using project db properly
This commit is contained in:
parent
e5db5cb4bb
commit
a385b01d4a
4 changed files with 119 additions and 119 deletions
|
@ -9,6 +9,7 @@ use InfluxDB\Database as InfluxDatabase;
|
||||||
use Utopia\App;
|
use Utopia\App;
|
||||||
use Utopia\CLI\Console;
|
use Utopia\CLI\Console;
|
||||||
use Utopia\Database\Database as UtopiaDatabase;
|
use Utopia\Database\Database as UtopiaDatabase;
|
||||||
|
use Utopia\Database\Document;
|
||||||
use Utopia\Database\Validator\Authorization;
|
use Utopia\Database\Validator\Authorization;
|
||||||
use Utopia\Logger\Log;
|
use Utopia\Logger\Log;
|
||||||
use Utopia\Validator\WhiteList;
|
use Utopia\Validator\WhiteList;
|
||||||
|
@ -50,10 +51,10 @@ $logError = function (Throwable $error, string $action = 'syncUsageStats') use (
|
||||||
Console::warning($error->getTraceAsString());
|
Console::warning($error->getTraceAsString());
|
||||||
};
|
};
|
||||||
|
|
||||||
function aggregateTimeseries(UtopiaDatabase $database, InfluxDatabase $influxDB, callable $logError): void
|
function aggregateTimeseries(UtopiaDatabase $database, InfluxDatabase $influxDB, callable $getProjectDB, callable $logError): void
|
||||||
{
|
{
|
||||||
$interval = (int) App::getEnv('_APP_USAGE_TIMESERIES_INTERVAL', '30'); // 30 seconds (by default)
|
$interval = (int) App::getEnv('_APP_USAGE_TIMESERIES_INTERVAL', '30'); // 30 seconds (by default)
|
||||||
$usage = new TimeSeries($database, $influxDB, $logError);
|
$usage = new TimeSeries($database, $influxDB, $getProjectDB, $logError);
|
||||||
|
|
||||||
Console::loop(function () use ($interval, $usage) {
|
Console::loop(function () use ($interval, $usage) {
|
||||||
$now = date('d-m-Y H:i:s', time());
|
$now = date('d-m-Y H:i:s', time());
|
||||||
|
@ -68,11 +69,11 @@ function aggregateTimeseries(UtopiaDatabase $database, InfluxDatabase $influxDB,
|
||||||
}, $interval);
|
}, $interval);
|
||||||
}
|
}
|
||||||
|
|
||||||
function aggregateDatabase(UtopiaDatabase $database, callable $logError): void
|
function aggregateDatabase(UtopiaDatabase $database, callable $getProjectDB, callable $logError): void
|
||||||
{
|
{
|
||||||
$interval = (int) App::getEnv('_APP_USAGE_DATABASE_INTERVAL', '900'); // 15 minutes (by default)
|
$interval = (int) App::getEnv('_APP_USAGE_DATABASE_INTERVAL', '900'); // 15 minutes (by default)
|
||||||
$usage = new Database($database, $logError);
|
$usage = new Database($database, $getProjectDB, $logError);
|
||||||
$aggregrator = new Aggregator($database, $logError);
|
$aggregrator = new Aggregator($database, $getProjectDB, $logError);
|
||||||
|
|
||||||
Console::loop(function () use ($interval, $usage, $aggregrator) {
|
Console::loop(function () use ($interval, $usage, $aggregrator) {
|
||||||
$now = date('d-m-Y H:i:s', time());
|
$now = date('d-m-Y H:i:s', time());
|
||||||
|
@ -97,13 +98,14 @@ $cli
|
||||||
|
|
||||||
$database = getConsoleDB();
|
$database = getConsoleDB();
|
||||||
$influxDB = getInfluxDB();
|
$influxDB = getInfluxDB();
|
||||||
|
$getProjectDB = fn (Document $project) => getProjectDB($project);
|
||||||
|
|
||||||
switch ($type) {
|
switch ($type) {
|
||||||
case 'timeseries':
|
case 'timeseries':
|
||||||
aggregateTimeseries($database, $influxDB, $logError);
|
aggregateTimeseries($database, $influxDB, $getProjectDB, $logError);
|
||||||
break;
|
break;
|
||||||
case 'database':
|
case 'database':
|
||||||
aggregateDatabase($database, $logError);
|
aggregateDatabase($database, $getProjectDB, $logError);
|
||||||
break;
|
break;
|
||||||
default:
|
default:
|
||||||
Console::error("Unsupported usage aggregation type");
|
Console::error("Unsupported usage aggregation type");
|
||||||
|
|
|
@ -9,10 +9,8 @@ use Utopia\Database\Query;
|
||||||
|
|
||||||
class Aggregator extends Database
|
class Aggregator extends Database
|
||||||
{
|
{
|
||||||
protected function aggregateDatabaseMetrics(string $projectId): void
|
protected function aggregateDatabaseMetrics(Document $project): void
|
||||||
{
|
{
|
||||||
$this->database->setNamespace('_' . $projectId);
|
|
||||||
|
|
||||||
$databasesGeneralMetrics = [
|
$databasesGeneralMetrics = [
|
||||||
'databases.$all.requests.create',
|
'databases.$all.requests.create',
|
||||||
'databases.$all.requests.read',
|
'databases.$all.requests.read',
|
||||||
|
@ -29,8 +27,8 @@ class Aggregator extends Database
|
||||||
];
|
];
|
||||||
|
|
||||||
foreach ($databasesGeneralMetrics as $metric) {
|
foreach ($databasesGeneralMetrics as $metric) {
|
||||||
$this->aggregateDailyMetric($projectId, $metric);
|
$this->aggregateDailyMetric($project, $metric);
|
||||||
$this->aggregateMonthlyMetric($projectId, $metric);
|
$this->aggregateMonthlyMetric($project, $metric);
|
||||||
}
|
}
|
||||||
|
|
||||||
$databasesDatabaseMetrics = [
|
$databasesDatabaseMetrics = [
|
||||||
|
@ -44,12 +42,12 @@ class Aggregator extends Database
|
||||||
'documents.databaseId.requests.delete',
|
'documents.databaseId.requests.delete',
|
||||||
];
|
];
|
||||||
|
|
||||||
$this->foreachDocument($projectId, 'databases', [], function (Document $database) use ($databasesDatabaseMetrics, $projectId) {
|
$this->foreachDocument($project, 'databases', [], function (Document $database) use ($databasesDatabaseMetrics, $project) {
|
||||||
$databaseId = $database->getId();
|
$databaseId = $database->getId();
|
||||||
foreach ($databasesDatabaseMetrics as $metric) {
|
foreach ($databasesDatabaseMetrics as $metric) {
|
||||||
$metric = str_replace('databaseId', $databaseId, $metric);
|
$metric = str_replace('databaseId', $databaseId, $metric);
|
||||||
$this->aggregateDailyMetric($projectId, $metric);
|
$this->aggregateDailyMetric($project, $metric);
|
||||||
$this->aggregateMonthlyMetric($projectId, $metric);
|
$this->aggregateMonthlyMetric($project, $metric);
|
||||||
}
|
}
|
||||||
|
|
||||||
$databasesCollectionMetrics = [
|
$databasesCollectionMetrics = [
|
||||||
|
@ -59,21 +57,19 @@ class Aggregator extends Database
|
||||||
'documents.' . $databaseId . '/collectionId.requests.delete',
|
'documents.' . $databaseId . '/collectionId.requests.delete',
|
||||||
];
|
];
|
||||||
|
|
||||||
$this->foreachDocument($projectId, 'database_' . $database->getInternalId(), [], function (Document $collection) use ($databasesCollectionMetrics, $projectId) {
|
$this->foreachDocument($project, 'database_' . $database->getInternalId(), [], function (Document $collection) use ($databasesCollectionMetrics, $project) {
|
||||||
$collectionId = $collection->getId();
|
$collectionId = $collection->getId();
|
||||||
foreach ($databasesCollectionMetrics as $metric) {
|
foreach ($databasesCollectionMetrics as $metric) {
|
||||||
$metric = str_replace('collectionId', $collectionId, $metric);
|
$metric = str_replace('collectionId', $collectionId, $metric);
|
||||||
$this->aggregateDailyMetric($projectId, $metric);
|
$this->aggregateDailyMetric($project, $metric);
|
||||||
$this->aggregateMonthlyMetric($projectId, $metric);
|
$this->aggregateMonthlyMetric($project, $metric);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
protected function aggregateStorageMetrics(string $projectId): void
|
protected function aggregateStorageMetrics(Document $project): void
|
||||||
{
|
{
|
||||||
$this->database->setNamespace('_' . $projectId);
|
|
||||||
|
|
||||||
$storageGeneralMetrics = [
|
$storageGeneralMetrics = [
|
||||||
'buckets.$all.requests.create',
|
'buckets.$all.requests.create',
|
||||||
'buckets.$all.requests.read',
|
'buckets.$all.requests.read',
|
||||||
|
@ -86,8 +82,8 @@ class Aggregator extends Database
|
||||||
];
|
];
|
||||||
|
|
||||||
foreach ($storageGeneralMetrics as $metric) {
|
foreach ($storageGeneralMetrics as $metric) {
|
||||||
$this->aggregateDailyMetric($projectId, $metric);
|
$this->aggregateDailyMetric($project, $metric);
|
||||||
$this->aggregateMonthlyMetric($projectId, $metric);
|
$this->aggregateMonthlyMetric($project, $metric);
|
||||||
}
|
}
|
||||||
|
|
||||||
$storageBucketMetrics = [
|
$storageBucketMetrics = [
|
||||||
|
@ -97,20 +93,18 @@ class Aggregator extends Database
|
||||||
'files.bucketId.requests.delete',
|
'files.bucketId.requests.delete',
|
||||||
];
|
];
|
||||||
|
|
||||||
$this->foreachDocument($projectId, 'buckets', [], function (Document $bucket) use ($storageBucketMetrics, $projectId) {
|
$this->foreachDocument($project, 'buckets', [], function (Document $bucket) use ($storageBucketMetrics, $project) {
|
||||||
$bucketId = $bucket->getId();
|
$bucketId = $bucket->getId();
|
||||||
foreach ($storageBucketMetrics as $metric) {
|
foreach ($storageBucketMetrics as $metric) {
|
||||||
$metric = str_replace('bucketId', $bucketId, $metric);
|
$metric = str_replace('bucketId', $bucketId, $metric);
|
||||||
$this->aggregateDailyMetric($projectId, $metric);
|
$this->aggregateDailyMetric($project, $metric);
|
||||||
$this->aggregateMonthlyMetric($projectId, $metric);
|
$this->aggregateMonthlyMetric($project, $metric);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
protected function aggregateFunctionMetrics(string $projectId): void
|
protected function aggregateFunctionMetrics(Document $project): void
|
||||||
{
|
{
|
||||||
$this->database->setNamespace('_' . $projectId);
|
|
||||||
|
|
||||||
$functionsGeneralMetrics = [
|
$functionsGeneralMetrics = [
|
||||||
'project.$all.compute.total',
|
'project.$all.compute.total',
|
||||||
'project.$all.compute.time',
|
'project.$all.compute.time',
|
||||||
|
@ -125,8 +119,8 @@ class Aggregator extends Database
|
||||||
];
|
];
|
||||||
|
|
||||||
foreach ($functionsGeneralMetrics as $metric) {
|
foreach ($functionsGeneralMetrics as $metric) {
|
||||||
$this->aggregateDailyMetric($projectId, $metric);
|
$this->aggregateDailyMetric($project, $metric);
|
||||||
$this->aggregateMonthlyMetric($projectId, $metric);
|
$this->aggregateMonthlyMetric($project, $metric);
|
||||||
}
|
}
|
||||||
|
|
||||||
$functionMetrics = [
|
$functionMetrics = [
|
||||||
|
@ -140,17 +134,17 @@ class Aggregator extends Database
|
||||||
'builds.functionId.compute.time',
|
'builds.functionId.compute.time',
|
||||||
];
|
];
|
||||||
|
|
||||||
$this->foreachDocument($projectId, 'functions', [], function (Document $function) use ($functionMetrics, $projectId) {
|
$this->foreachDocument($project, 'functions', [], function (Document $function) use ($functionMetrics, $project) {
|
||||||
$functionId = $function->getId();
|
$functionId = $function->getId();
|
||||||
foreach ($functionMetrics as $metric) {
|
foreach ($functionMetrics as $metric) {
|
||||||
$metric = str_replace('functionId', $functionId, $metric);
|
$metric = str_replace('functionId', $functionId, $metric);
|
||||||
$this->aggregateDailyMetric($projectId, $metric);
|
$this->aggregateDailyMetric($project, $metric);
|
||||||
$this->aggregateMonthlyMetric($projectId, $metric);
|
$this->aggregateMonthlyMetric($project, $metric);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
protected function aggregateUsersMetrics(string $projectId): void
|
protected function aggregateUsersMetrics(Document $project): void
|
||||||
{
|
{
|
||||||
$metrics = [
|
$metrics = [
|
||||||
'users.$all.requests.create',
|
'users.$all.requests.create',
|
||||||
|
@ -162,50 +156,50 @@ class Aggregator extends Database
|
||||||
];
|
];
|
||||||
|
|
||||||
foreach ($metrics as $metric) {
|
foreach ($metrics as $metric) {
|
||||||
$this->aggregateDailyMetric($projectId, $metric);
|
$this->aggregateDailyMetric($project, $metric);
|
||||||
$this->aggregateMonthlyMetric($projectId, $metric);
|
$this->aggregateMonthlyMetric($project, $metric);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
protected function aggregateGeneralMetrics(string $projectId): void
|
protected function aggregateGeneralMetrics(Document $project): void
|
||||||
{
|
{
|
||||||
$this->aggregateDailyMetric($projectId, 'project.$all.network.requests');
|
$this->aggregateDailyMetric($project, 'project.$all.network.requests');
|
||||||
$this->aggregateDailyMetric($projectId, 'project.$all.network.bandwidth');
|
$this->aggregateDailyMetric($project, 'project.$all.network.bandwidth');
|
||||||
$this->aggregateDailyMetric($projectId, 'project.$all.network.inbound');
|
$this->aggregateDailyMetric($project, 'project.$all.network.inbound');
|
||||||
$this->aggregateDailyMetric($projectId, 'project.$all.network.outbound');
|
$this->aggregateDailyMetric($project, 'project.$all.network.outbound');
|
||||||
$this->aggregateMonthlyMetric($projectId, 'project.$all.network.requests');
|
$this->aggregateMonthlyMetric($project, 'project.$all.network.requests');
|
||||||
$this->aggregateMonthlyMetric($projectId, 'project.$all.network.bandwidth');
|
$this->aggregateMonthlyMetric($project, 'project.$all.network.bandwidth');
|
||||||
$this->aggregateMonthlyMetric($projectId, 'project.$all.network.inbound');
|
$this->aggregateMonthlyMetric($project, 'project.$all.network.inbound');
|
||||||
$this->aggregateMonthlyMetric($projectId, 'project.$all.network.outbound');
|
$this->aggregateMonthlyMetric($project, 'project.$all.network.outbound');
|
||||||
}
|
}
|
||||||
|
|
||||||
protected function aggregateDailyMetric(string $projectId, string $metric): void
|
protected function aggregateDailyMetric(Document $project, string $metric): void
|
||||||
{
|
{
|
||||||
$beginOfDay = DateTime::createFromFormat('Y-m-d\TH:i:s.v', \date('Y-m-d\T00:00:00.000'))->format(DateTime::RFC3339);
|
$beginOfDay = DateTime::createFromFormat('Y-m-d\TH:i:s.v', \date('Y-m-d\T00:00:00.000'))->format(DateTime::RFC3339);
|
||||||
$endOfDay = DateTime::createFromFormat('Y-m-d\TH:i:s.v', \date('Y-m-d\T23:59:59.999'))->format(DateTime::RFC3339);
|
$endOfDay = DateTime::createFromFormat('Y-m-d\TH:i:s.v', \date('Y-m-d\T23:59:59.999'))->format(DateTime::RFC3339);
|
||||||
|
|
||||||
$this->database->setNamespace('_' . $projectId);
|
$database = call_user_func($this->getProjectDB, $project);
|
||||||
$value = (int) $this->database->sum('stats', 'value', [
|
$value = (int) $database->sum('stats', 'value', [
|
||||||
Query::equal('metric', [$metric]),
|
Query::equal('metric', [$metric]),
|
||||||
Query::equal('period', ['30m']),
|
Query::equal('period', ['30m']),
|
||||||
Query::greaterThanEqual('time', $beginOfDay),
|
Query::greaterThanEqual('time', $beginOfDay),
|
||||||
Query::lessThanEqual('time', $endOfDay),
|
Query::lessThanEqual('time', $endOfDay),
|
||||||
]);
|
]);
|
||||||
$this->createOrUpdateMetric($projectId, $metric, '1d', $beginOfDay, $value);
|
$this->createOrUpdateMetric($database, $project->getId(), $metric, '1d', $beginOfDay, $value);
|
||||||
}
|
}
|
||||||
|
|
||||||
protected function aggregateMonthlyMetric(string $projectId, string $metric): void
|
protected function aggregateMonthlyMetric(Document $project, string $metric): void
|
||||||
{
|
{
|
||||||
$beginOfMonth = DateTime::createFromFormat('Y-m-d\TH:i:s.v', \date('Y-m-01\T00:00:00.000'))->format(DateTime::RFC3339);
|
$beginOfMonth = DateTime::createFromFormat('Y-m-d\TH:i:s.v', \date('Y-m-01\T00:00:00.000'))->format(DateTime::RFC3339);
|
||||||
$endOfMonth = DateTime::createFromFormat('Y-m-d\TH:i:s.v', \date('Y-m-t\T23:59:59.999'))->format(DateTime::RFC3339);
|
$endOfMonth = DateTime::createFromFormat('Y-m-d\TH:i:s.v', \date('Y-m-t\T23:59:59.999'))->format(DateTime::RFC3339);
|
||||||
$this->database->setNamespace('_' . $projectId);
|
$database = call_user_func($this->getProjectDB, $project);
|
||||||
$value = (int) $this->database->sum('stats', 'value', [
|
$value = (int) $database->sum('stats', 'value', [
|
||||||
Query::equal('metric', [$metric]),
|
Query::equal('metric', [$metric]),
|
||||||
Query::equal('period', ['1d']),
|
Query::equal('period', ['1d']),
|
||||||
Query::greaterThanEqual('time', $beginOfMonth),
|
Query::greaterThanEqual('time', $beginOfMonth),
|
||||||
Query::lessThanEqual('time', $endOfMonth),
|
Query::lessThanEqual('time', $endOfMonth),
|
||||||
]);
|
]);
|
||||||
$this->createOrUpdateMetric($projectId, $metric, '1mo', $beginOfMonth, $value);
|
$this->createOrUpdateMetric($database, $project->getId(), $metric, '1mo', $beginOfMonth, $value);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -216,16 +210,12 @@ class Aggregator extends Database
|
||||||
*/
|
*/
|
||||||
public function collect(): void
|
public function collect(): void
|
||||||
{
|
{
|
||||||
$this->foreachDocument('console', 'projects', [], function (Document $project) {
|
$this->foreachDocument(new Document(['$id' => 'console']), 'projects', [], function (Document $project) {
|
||||||
$projectId = $project->getInternalId();
|
$this->aggregateGeneralMetrics($project);
|
||||||
|
$this->aggregateFunctionMetrics($project);
|
||||||
// Aggregate new metrics from already collected usage metrics
|
$this->aggregateDatabaseMetrics($project);
|
||||||
// for lower time period (1day and 1 month metric from 30 minute metrics)
|
$this->aggregateStorageMetrics($project);
|
||||||
$this->aggregateGeneralMetrics($projectId);
|
$this->aggregateUsersMetrics($project);
|
||||||
$this->aggregateFunctionMetrics($projectId);
|
|
||||||
$this->aggregateDatabaseMetrics($projectId);
|
|
||||||
$this->aggregateStorageMetrics($projectId);
|
|
||||||
$this->aggregateUsersMetrics($projectId);
|
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -24,9 +24,10 @@ class Database extends Calculator
|
||||||
],
|
],
|
||||||
];
|
];
|
||||||
|
|
||||||
public function __construct(UtopiaDatabase $database, callable $errorHandler = null)
|
public function __construct(UtopiaDatabase $database, callable $getProjectDB, callable $errorHandler = null)
|
||||||
{
|
{
|
||||||
$this->database = $database;
|
$this->database = $database;
|
||||||
|
$this->getProjectDB = $getProjectDB;
|
||||||
$this->errorHandler = $errorHandler;
|
$this->errorHandler = $errorHandler;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -35,7 +36,8 @@ class Database extends Calculator
|
||||||
*
|
*
|
||||||
* Create given metric for each defined period
|
* Create given metric for each defined period
|
||||||
*
|
*
|
||||||
* @param string $projectId
|
* @param UtopiaDatabase $database
|
||||||
|
* @param Document $project
|
||||||
* @param string $metric
|
* @param string $metric
|
||||||
* @param int $value
|
* @param int $value
|
||||||
* @param bool $monthly
|
* @param bool $monthly
|
||||||
|
@ -43,7 +45,7 @@ class Database extends Calculator
|
||||||
* @throws Authorization
|
* @throws Authorization
|
||||||
* @throws Structure
|
* @throws Structure
|
||||||
*/
|
*/
|
||||||
protected function createPerPeriodMetric(string $projectId, string $metric, int $value, bool $monthly = false): void
|
protected function createPerPeriodMetric(UtopiaDatabase $database, string $projectId, string $metric, int $value, bool $monthly = false): void
|
||||||
{
|
{
|
||||||
foreach ($this->periods as $options) {
|
foreach ($this->periods as $options) {
|
||||||
$period = $options['key'];
|
$period = $options['key'];
|
||||||
|
@ -56,13 +58,13 @@ class Database extends Calculator
|
||||||
} else {
|
} else {
|
||||||
throw new Exception("Period type not found", 500);
|
throw new Exception("Period type not found", 500);
|
||||||
}
|
}
|
||||||
$this->createOrUpdateMetric($projectId, $metric, $period, $time, $value);
|
$this->createOrUpdateMetric($database, $projectId, $metric, $period, $time, $value);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Required for billing
|
// Required for billing
|
||||||
if ($monthly) {
|
if ($monthly) {
|
||||||
$time = DateTime::createFromFormat('Y-m-d\TH:i:s.v', \date('Y-m-01\T00:00:00.000'))->format(DateTime::RFC3339);
|
$time = DateTime::createFromFormat('Y-m-d\TH:i:s.v', \date('Y-m-01\T00:00:00.000'))->format(DateTime::RFC3339);
|
||||||
$this->createOrUpdateMetric($projectId, $metric, '1mo', $time, $value);
|
$this->createOrUpdateMetric($database, $projectId, $metric, '1mo', $time, $value);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -71,7 +73,8 @@ class Database extends Calculator
|
||||||
*
|
*
|
||||||
* Create or update each metric in the stats collection for the given project
|
* Create or update each metric in the stats collection for the given project
|
||||||
*
|
*
|
||||||
* @param string $projectId
|
* @param UtopiaDatabase $database
|
||||||
|
* @param String $projectId
|
||||||
* @param string $metric
|
* @param string $metric
|
||||||
* @param string $period
|
* @param string $period
|
||||||
* @param string $time
|
* @param string $time
|
||||||
|
@ -81,15 +84,14 @@ class Database extends Calculator
|
||||||
* @throws Authorization
|
* @throws Authorization
|
||||||
* @throws Structure
|
* @throws Structure
|
||||||
*/
|
*/
|
||||||
protected function createOrUpdateMetric(string $projectId, string $metric, string $period, string $time, int $value): void
|
protected function createOrUpdateMetric(UtopiaDatabase $database, String $projectId, string $metric, string $period, string $time, int $value): void
|
||||||
{
|
{
|
||||||
$id = \md5("{$time}_{$period}_{$metric}");
|
$id = \md5("{$time}_{$period}_{$metric}");
|
||||||
$this->database->setNamespace('_' . $projectId);
|
|
||||||
|
|
||||||
try {
|
try {
|
||||||
$document = $this->database->getDocument('stats', $id);
|
$document = $database->getDocument('stats', $id);
|
||||||
if ($document->isEmpty()) {
|
if ($document->isEmpty()) {
|
||||||
$this->database->createDocument('stats', new Document([
|
$database->createDocument('stats', new Document([
|
||||||
'$id' => $id,
|
'$id' => $id,
|
||||||
'period' => $period,
|
'period' => $period,
|
||||||
'time' => $time,
|
'time' => $time,
|
||||||
|
@ -98,7 +100,7 @@ class Database extends Calculator
|
||||||
'type' => 2, // these are cumulative metrics
|
'type' => 2, // these are cumulative metrics
|
||||||
]));
|
]));
|
||||||
} else {
|
} else {
|
||||||
$this->database->updateDocument(
|
$database->updateDocument(
|
||||||
'stats',
|
'stats',
|
||||||
$document->getId(),
|
$document->getId(),
|
||||||
$document->setAttribute('value', $value)
|
$document->setAttribute('value', $value)
|
||||||
|
@ -118,7 +120,7 @@ class Database extends Calculator
|
||||||
*
|
*
|
||||||
* Call provided callback for each document in the collection
|
* Call provided callback for each document in the collection
|
||||||
*
|
*
|
||||||
* @param string $projectId
|
* @param Document $project
|
||||||
* @param string $collection
|
* @param string $collection
|
||||||
* @param array $queries
|
* @param array $queries
|
||||||
* @param callable $callback
|
* @param callable $callback
|
||||||
|
@ -126,13 +128,13 @@ class Database extends Calculator
|
||||||
* @return void
|
* @return void
|
||||||
* @throws Exception
|
* @throws Exception
|
||||||
*/
|
*/
|
||||||
protected function foreachDocument(string $projectId, string $collection, array $queries, callable $callback): void
|
protected function foreachDocument(Document $project, string $collection, array $queries, callable $callback): void
|
||||||
{
|
{
|
||||||
$limit = 50;
|
$limit = 50;
|
||||||
$results = [];
|
$results = [];
|
||||||
$sum = $limit;
|
$sum = $limit;
|
||||||
$latestDocument = null;
|
$latestDocument = null;
|
||||||
$this->database->setNamespace($projectId === 'console' ? $projectId : '_' . $projectId);
|
$database = $project->getId() == 'console' ? $this->database : call_user_func($this->getProjectDB, $project);
|
||||||
|
|
||||||
while ($sum === $limit) {
|
while ($sum === $limit) {
|
||||||
try {
|
try {
|
||||||
|
@ -143,7 +145,7 @@ class Database extends Calculator
|
||||||
$results = $this->database->find($collection, \array_merge($paginationQueries, $queries));
|
$results = $this->database->find($collection, \array_merge($paginationQueries, $queries));
|
||||||
} catch (\Exception $e) {
|
} catch (\Exception $e) {
|
||||||
if (is_callable($this->errorHandler)) {
|
if (is_callable($this->errorHandler)) {
|
||||||
call_user_func($this->errorHandler, $e, "fetch_documents_project_{$projectId}_collection_{$collection}");
|
call_user_func($this->errorHandler, $e, "fetch_documents_project_{$project->getId()}_collection_{$collection}");
|
||||||
return;
|
return;
|
||||||
} else {
|
} else {
|
||||||
throw $e;
|
throw $e;
|
||||||
|
@ -169,6 +171,7 @@ class Database extends Calculator
|
||||||
*
|
*
|
||||||
* Calculate sum of an attribute of documents in collection
|
* Calculate sum of an attribute of documents in collection
|
||||||
*
|
*
|
||||||
|
* @param UtopiaDatabase $database
|
||||||
* @param string $projectId
|
* @param string $projectId
|
||||||
* @param string $collection
|
* @param string $collection
|
||||||
* @param string $attribute
|
* @param string $attribute
|
||||||
|
@ -177,16 +180,15 @@ class Database extends Calculator
|
||||||
* @return int
|
* @return int
|
||||||
* @throws Exception
|
* @throws Exception
|
||||||
*/
|
*/
|
||||||
private function sum(string $projectId, string $collection, string $attribute, string $metric = null, int $multiplier = 1): int
|
private function sum(UtopiaDatabase $database, string $projectId, string $collection, string $attribute, string $metric = null, int $multiplier = 1): int
|
||||||
{
|
{
|
||||||
$this->database->setNamespace('_' . $projectId);
|
|
||||||
|
|
||||||
try {
|
try {
|
||||||
$sum = $this->database->sum($collection, $attribute);
|
$sum = $database->sum($collection, $attribute);
|
||||||
$sum = (int) ($sum * $multiplier);
|
$sum = (int) ($sum * $multiplier);
|
||||||
|
|
||||||
if (!is_null($metric)) {
|
if (!is_null($metric)) {
|
||||||
$this->createPerPeriodMetric($projectId, $metric, $sum);
|
$this->createPerPeriodMetric($database, $projectId, $metric, $sum);
|
||||||
}
|
}
|
||||||
return $sum;
|
return $sum;
|
||||||
} catch (Exception $e) {
|
} catch (Exception $e) {
|
||||||
|
@ -204,6 +206,7 @@ class Database extends Calculator
|
||||||
*
|
*
|
||||||
* Count number of documents in collection
|
* Count number of documents in collection
|
||||||
*
|
*
|
||||||
|
* @param UtopiaDatabase $database
|
||||||
* @param string $projectId
|
* @param string $projectId
|
||||||
* @param string $collection
|
* @param string $collection
|
||||||
* @param ?string $metric
|
* @param ?string $metric
|
||||||
|
@ -211,14 +214,14 @@ class Database extends Calculator
|
||||||
* @return int
|
* @return int
|
||||||
* @throws Exception
|
* @throws Exception
|
||||||
*/
|
*/
|
||||||
private function count(string $projectId, string $collection, ?string $metric = null): int
|
private function count(UtopiaDatabase $database, string $projectId, string $collection, ?string $metric = null): int
|
||||||
{
|
{
|
||||||
$this->database->setNamespace('_' . $projectId);
|
$this->database->setNamespace('_' . $projectId);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
$count = $this->database->count($collection);
|
$count = $this->database->count($collection);
|
||||||
if (!is_null($metric)) {
|
if (!is_null($metric)) {
|
||||||
$this->createPerPeriodMetric($projectId, (string) $metric, $count);
|
$this->createPerPeriodMetric($database, $projectId, (string) $metric, $count);
|
||||||
}
|
}
|
||||||
return $count;
|
return $count;
|
||||||
} catch (Exception $e) {
|
} catch (Exception $e) {
|
||||||
|
@ -236,14 +239,15 @@ class Database extends Calculator
|
||||||
*
|
*
|
||||||
* Total sum of storage used by deployments
|
* Total sum of storage used by deployments
|
||||||
*
|
*
|
||||||
|
* @param UtopiaDatabase $database
|
||||||
* @param string $projectId
|
* @param string $projectId
|
||||||
*
|
*
|
||||||
* @return int
|
* @return int
|
||||||
* @throws Exception
|
* @throws Exception
|
||||||
*/
|
*/
|
||||||
private function deploymentsTotal(string $projectId): int
|
private function deploymentsTotal(UtopiaDatabase $database, string $projectId): int
|
||||||
{
|
{
|
||||||
return $this->sum($projectId, 'deployments', 'size', 'deployments.$all.storage.size');
|
return $this->sum($database, $projectId, 'deployments', 'size', 'deployments.$all.storage.size');
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -251,14 +255,15 @@ class Database extends Calculator
|
||||||
*
|
*
|
||||||
* Metric: users.count
|
* Metric: users.count
|
||||||
*
|
*
|
||||||
|
* @param UtopiaDatabase $database
|
||||||
* @param string $projectId
|
* @param string $projectId
|
||||||
*
|
*
|
||||||
* @return void
|
* @return void
|
||||||
* @throws Exception
|
* @throws Exception
|
||||||
*/
|
*/
|
||||||
private function usersStats(string $projectId): void
|
private function usersStats(UtopiaDatabase $database, string $projectId): void
|
||||||
{
|
{
|
||||||
$this->count($projectId, 'users', 'users.$all.count.total');
|
$this->count($database, $projectId, 'users', 'users.$all.count.total');
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -267,35 +272,36 @@ class Database extends Calculator
|
||||||
* Metrics: buckets.$all.count.total, files.$all.count.total, files.bucketId,count.total,
|
* Metrics: buckets.$all.count.total, files.$all.count.total, files.bucketId,count.total,
|
||||||
* files.$all.storage.size, files.bucketId.storage.size, project.$all.storage.size
|
* files.$all.storage.size, files.bucketId.storage.size, project.$all.storage.size
|
||||||
*
|
*
|
||||||
* @param string $projectId
|
* @param UtopiaDatabase $database
|
||||||
|
* @param Document $project
|
||||||
*
|
*
|
||||||
* @return void
|
* @return void
|
||||||
* @throws Authorization
|
* @throws Authorization
|
||||||
* @throws Structure
|
* @throws Structure
|
||||||
*/
|
*/
|
||||||
private function storageStats(string $projectId): void
|
private function storageStats(UtopiaDatabase $database, Document $project): void
|
||||||
{
|
{
|
||||||
$projectFilesTotal = 0;
|
$projectFilesTotal = 0;
|
||||||
$projectFilesCount = 0;
|
$projectFilesCount = 0;
|
||||||
|
|
||||||
$metric = 'buckets.$all.count.total';
|
$metric = 'buckets.$all.count.total';
|
||||||
$this->count($projectId, 'buckets', $metric);
|
$this->count($database, $project->getId(), 'buckets', $metric);
|
||||||
|
|
||||||
$this->foreachDocument($projectId, 'buckets', [], function ($bucket) use (&$projectFilesCount, &$projectFilesTotal, $projectId,) {
|
$this->foreachDocument($project, 'buckets', [], function ($bucket) use (&$projectFilesCount, &$projectFilesTotal, $project, $database) {
|
||||||
$metric = "files.{$bucket->getId()}.count.total";
|
$metric = "files.{$bucket->getId()}.count.total";
|
||||||
$count = $this->count($projectId, 'bucket_' . $bucket->getInternalId(), $metric);
|
$count = $this->count($database, $project->getId(), 'bucket_' . $bucket->getInternalId(), $metric);
|
||||||
$projectFilesCount += $count;
|
$projectFilesCount += $count;
|
||||||
|
|
||||||
$metric = "files.{$bucket->getId()}.storage.size";
|
$metric = "files.{$bucket->getId()}.storage.size";
|
||||||
$sum = $this->sum($projectId, 'bucket_' . $bucket->getInternalId(), 'sizeOriginal', $metric);
|
$sum = $this->sum($database, $project->getId(), 'bucket_' . $bucket->getInternalId(), 'sizeOriginal', $metric);
|
||||||
$projectFilesTotal += $sum;
|
$projectFilesTotal += $sum;
|
||||||
});
|
});
|
||||||
|
|
||||||
$this->createPerPeriodMetric($projectId, 'files.$all.count.total', $projectFilesCount);
|
$this->createPerPeriodMetric($database, $project->getId(), 'files.$all.count.total', $projectFilesCount);
|
||||||
$this->createPerPeriodMetric($projectId, 'files.$all.storage.size', $projectFilesTotal);
|
$this->createPerPeriodMetric($database, $project->getId(), 'files.$all.storage.size', $projectFilesTotal);
|
||||||
|
|
||||||
$deploymentsTotal = $this->deploymentsTotal($projectId);
|
$deploymentsTotal = $this->deploymentsTotal($database, $project->getId());
|
||||||
$this->createPerPeriodMetric($projectId, 'project.$all.storage.size', $projectFilesTotal + $deploymentsTotal);
|
$this->createPerPeriodMetric($database, $project->getId(), 'project.$all.storage.size', $projectFilesTotal + $deploymentsTotal);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -305,38 +311,39 @@ class Database extends Calculator
|
||||||
* Metrics: databases.$all.count.total, collections.$all.count.total, collections.databaseId.count.total,
|
* Metrics: databases.$all.count.total, collections.$all.count.total, collections.databaseId.count.total,
|
||||||
* documents.$all.count.all, documents.databaseId.count.total, documents.databaseId/collectionId.count.total
|
* documents.$all.count.all, documents.databaseId.count.total, documents.databaseId/collectionId.count.total
|
||||||
*
|
*
|
||||||
* @param string $projectId
|
* @param UtopiaDatabase $database
|
||||||
|
* @param Document $project
|
||||||
*
|
*
|
||||||
* @return void
|
* @return void
|
||||||
* @throws Authorization
|
* @throws Authorization
|
||||||
* @throws Structure
|
* @throws Structure
|
||||||
*/
|
*/
|
||||||
private function databaseStats(string $projectId): void
|
private function databaseStats(UtopiaDatabase $database, Document $project): void
|
||||||
{
|
{
|
||||||
$projectDocumentsCount = 0;
|
$projectDocumentsCount = 0;
|
||||||
$projectCollectionsCount = 0;
|
$projectCollectionsCount = 0;
|
||||||
|
|
||||||
$this->count($projectId, 'databases', 'databases.$all.count.total');
|
$this->count($database, $project->getId(), 'databases', 'databases.$all.count.total');
|
||||||
|
|
||||||
$this->foreachDocument($projectId, 'databases', [], function ($database) use (&$projectDocumentsCount, &$projectCollectionsCount, $projectId) {
|
$this->foreachDocument($project, 'databases', [], function ($database) use (&$projectDocumentsCount, &$projectCollectionsCount, $project) {
|
||||||
$metric = "collections.{$database->getId()}.count.total";
|
$metric = "collections.{$database->getId()}.count.total";
|
||||||
$count = $this->count($projectId, 'database_' . $database->getInternalId(), $metric);
|
$count = $this->count($database, $project->getId(), 'database_' . $database->getInternalId(), $metric);
|
||||||
$projectCollectionsCount += $count;
|
$projectCollectionsCount += $count;
|
||||||
$databaseDocumentsCount = 0;
|
$databaseDocumentsCount = 0;
|
||||||
|
|
||||||
$this->foreachDocument($projectId, 'database_' . $database->getInternalId(), [], function ($collection) use (&$projectDocumentsCount, &$databaseDocumentsCount, $projectId, $database) {
|
$this->foreachDocument($project, 'database_' . $database->getInternalId(), [], function ($collection) use (&$projectDocumentsCount, &$databaseDocumentsCount, $project, $database) {
|
||||||
$metric = "documents.{$database->getId()}/{$collection->getId()}.count.total";
|
$metric = "documents.{$database->getId()}/{$collection->getId()}.count.total";
|
||||||
|
|
||||||
$count = $this->count($projectId, 'database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $metric);
|
$count = $this->count($database, $project->getId(), 'database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $metric);
|
||||||
$projectDocumentsCount += $count;
|
$projectDocumentsCount += $count;
|
||||||
$databaseDocumentsCount += $count;
|
$databaseDocumentsCount += $count;
|
||||||
});
|
});
|
||||||
|
|
||||||
$this->createPerPeriodMetric($projectId, "documents.{$database->getId()}.count.total", $databaseDocumentsCount);
|
$this->createPerPeriodMetric($database, $project->getId(), "documents.{$database->getId()}.count.total", $databaseDocumentsCount);
|
||||||
});
|
});
|
||||||
|
|
||||||
$this->createPerPeriodMetric($projectId, 'collections.$all.count.total', $projectCollectionsCount);
|
$this->createPerPeriodMetric($database, $project->getId(), 'collections.$all.count.total', $projectCollectionsCount);
|
||||||
$this->createPerPeriodMetric($projectId, 'documents.$all.count.total', $projectDocumentsCount);
|
$this->createPerPeriodMetric($database, $project->getId(), 'documents.$all.count.total', $projectDocumentsCount);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -349,12 +356,11 @@ class Database extends Calculator
|
||||||
*/
|
*/
|
||||||
public function collect(): void
|
public function collect(): void
|
||||||
{
|
{
|
||||||
$this->foreachDocument('console', 'projects', [], function (Document $project) {
|
$this->foreachDocument(new Document(['$id' => 'console']), 'projects', [], function (Document $project) {
|
||||||
$projectId = $project->getInternalId();
|
$database = call_user_func($this->getProjectDB, $project);
|
||||||
|
$this->usersStats($database, $project->getId());
|
||||||
$this->usersStats($projectId);
|
$this->databaseStats($database, $project);
|
||||||
$this->databaseStats($projectId);
|
$this->storageStats($database, $project);
|
||||||
$this->storageStats($projectId);
|
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -14,6 +14,7 @@ class TimeSeries extends Calculator
|
||||||
protected Database $database;
|
protected Database $database;
|
||||||
protected $errorHandler;
|
protected $errorHandler;
|
||||||
private array $latestTime = [];
|
private array $latestTime = [];
|
||||||
|
private mixed $getProjectDB;
|
||||||
|
|
||||||
// all the mertics that we are collecting
|
// all the mertics that we are collecting
|
||||||
protected array $metrics = [
|
protected array $metrics = [
|
||||||
|
@ -278,10 +279,11 @@ class TimeSeries extends Calculator
|
||||||
'startTime' => '-24 hours',
|
'startTime' => '-24 hours',
|
||||||
];
|
];
|
||||||
|
|
||||||
public function __construct(Database $database, InfluxDatabase $influxDB, callable $errorHandler = null)
|
public function __construct(Database $database, InfluxDatabase $influxDB, callable $getProjectDB, callable $errorHandler = null)
|
||||||
{
|
{
|
||||||
$this->database = $database;
|
$this->database = $database;
|
||||||
$this->influxDB = $influxDB;
|
$this->influxDB = $influxDB;
|
||||||
|
$this->getProjectDB = $getProjectDB;
|
||||||
$this->errorHandler = $errorHandler;
|
$this->errorHandler = $errorHandler;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -303,10 +305,10 @@ class TimeSeries extends Calculator
|
||||||
$id = \md5("{$time}_{$period}_{$metric}");
|
$id = \md5("{$time}_{$period}_{$metric}");
|
||||||
$this->database->setNamespace('console');
|
$this->database->setNamespace('console');
|
||||||
$project = $this->database->getDocument('projects', $projectId);
|
$project = $this->database->getDocument('projects', $projectId);
|
||||||
$this->database->setNamespace('_' . $project->getInternalId());
|
$database = call_user_func($this->getProjectDB, $project);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
$document = $this->database->getDocument('stats', $id);
|
$document = $database->getDocument('stats', $id);
|
||||||
if ($document->isEmpty()) {
|
if ($document->isEmpty()) {
|
||||||
$this->database->createDocument('stats', new Document([
|
$this->database->createDocument('stats', new Document([
|
||||||
'$id' => $id,
|
'$id' => $id,
|
||||||
|
@ -317,7 +319,7 @@ class TimeSeries extends Calculator
|
||||||
'type' => $type,
|
'type' => $type,
|
||||||
]));
|
]));
|
||||||
} else {
|
} else {
|
||||||
$this->database->updateDocument(
|
$database->updateDocument(
|
||||||
'stats',
|
'stats',
|
||||||
$document->getId(),
|
$document->getId(),
|
||||||
$document->setAttribute('value', $value)
|
$document->setAttribute('value', $value)
|
||||||
|
|
Loading…
Reference in a new issue