deprecate database aggregation
This commit is contained in:
Damodar Lohani
parent
3ed3ef9d71
commit
8048d24857
3
.env
3
.env
|
|
@ -76,8 +76,7 @@ _APP_MAINTENANCE_RETENTION_CACHE=2592000
|
|||
_APP_MAINTENANCE_RETENTION_EXECUTION=1209600
|
||||
_APP_MAINTENANCE_RETENTION_ABUSE=86400
|
||||
_APP_MAINTENANCE_RETENTION_AUDIT=1209600
|
||||
_APP_USAGE_TIMESERIES_INTERVAL=2
|
||||
_APP_USAGE_DATABASE_INTERVAL=15
|
||||
_APP_USAGE_AGGREGATION_INTERVAL=5
|
||||
_APP_USAGE_STATS=enabled
|
||||
_APP_LOGGING_PROVIDER=
|
||||
_APP_LOGGING_CONFIG=
|
||||
|
|
|
|||
|
|
@ -246,8 +246,7 @@ ENV _APP_SERVER=swoole \
|
|||
_APP_SETUP=self-hosted \
|
||||
_APP_VERSION=$VERSION \
|
||||
_APP_USAGE_STATS=enabled \
|
||||
_APP_USAGE_TIMESERIES_INTERVAL=30 \
|
||||
_APP_USAGE_DATABASE_INTERVAL=900 \
|
||||
_APP_USAGE_AGGREGATION_INTERVAL=30 \
|
||||
# 14 Days = 1209600 s
|
||||
_APP_MAINTENANCE_RETENTION_EXECUTION=1209600 \
|
||||
_APP_MAINTENANCE_RETENTION_AUDIT=1209600 \
|
||||
|
|
|
|||
|
|
@ -179,7 +179,7 @@ return [
|
|||
],
|
||||
[
|
||||
'name' => '_APP_USAGE_TIMESERIES_INTERVAL',
|
||||
'description' => 'Interval value containing the number of seconds that the Appwrite usage process should wait before aggregating stats and syncing it to Appwrite Database from Timeseries Database. The default value is 30 seconds.',
|
||||
'description' => 'Deprecated since _ use _APP_USAGE_AGGREGATION_INTERVAL instead.',
|
||||
'introduction' => '1.0.0',
|
||||
'default' => '30',
|
||||
'required' => false,
|
||||
|
|
@ -188,7 +188,7 @@ return [
|
|||
],
|
||||
[
|
||||
'name' => '_APP_USAGE_DATABASE_INTERVAL',
|
||||
'description' => 'Interval value containing the number of seconds that the Appwrite usage process should wait before aggregating stats from data in Appwrite Database. The default value is 15 minutes.',
|
||||
'description' => 'Deprecated since _ use _APP_USAGE_AGGREGATION_INTERVAL instead.',
|
||||
'introduction' => '1.0.0',
|
||||
'default' => '900',
|
||||
'required' => false,
|
||||
|
|
|
|||
|
|
@ -114,44 +114,6 @@ $logError = function (Throwable $error, string $action = 'syncUsageStats') use (
|
|||
Console::warning($error->getTraceAsString());
|
||||
};
|
||||
|
||||
|
||||
function aggregateTimeseries(UtopiaDatabase $database, InfluxDatabase $influxDB, callable $logError): void
|
||||
{
|
||||
$interval = (int) App::getEnv('_APP_USAGE_TIMESERIES_INTERVAL', '30'); // 30 seconds (by default)
|
||||
$usage = new TimeSeries($database, $influxDB, $logError);
|
||||
|
||||
Console::loop(function () use ($interval, $usage) {
|
||||
$now = date('d-m-Y H:i:s', time());
|
||||
Console::info("[{$now}] Aggregating Timeseries Usage data every {$interval} seconds");
|
||||
$loopStart = microtime(true);
|
||||
|
||||
$usage->collect();
|
||||
|
||||
$loopTook = microtime(true) - $loopStart;
|
||||
$now = date('d-m-Y H:i:s', time());
|
||||
Console::info("[{$now}] Aggregation took {$loopTook} seconds");
|
||||
}, $interval);
|
||||
}
|
||||
|
||||
function aggregateDatabase(UtopiaDatabase $database, callable $logError): void
|
||||
{
|
||||
$interval = (int) App::getEnv('_APP_USAGE_DATABASE_INTERVAL', '900'); // 15 minutes (by default)
|
||||
$usage = new Database($database, $logError);
|
||||
$aggregrator = new Aggregator($database, $logError);
|
||||
|
||||
Console::loop(function () use ($interval, $usage, $aggregrator) {
|
||||
$now = date('d-m-Y H:i:s', time());
|
||||
Console::info("[{$now}] Aggregating database usage every {$interval} seconds.");
|
||||
$loopStart = microtime(true);
|
||||
$usage->collect();
|
||||
$aggregrator->collect();
|
||||
$loopTook = microtime(true) - $loopStart;
|
||||
$now = date('d-m-Y H:i:s', time());
|
||||
|
||||
Console::info("[{$now}] Aggregation took {$loopTook} seconds");
|
||||
}, $interval);
|
||||
}
|
||||
|
||||
$cli
|
||||
->task('usage')
|
||||
->param('type', 'timeseries', new WhiteList(['timeseries', 'database']))
|
||||
|
|
@ -163,14 +125,18 @@ $cli
|
|||
$database = getDatabase($register, '_console');
|
||||
$influxDB = getInfluxDB($register);
|
||||
|
||||
switch ($type) {
|
||||
case 'timeseries':
|
||||
aggregateTimeseries($database, $influxDB, $logError);
|
||||
break;
|
||||
case 'database':
|
||||
aggregateDatabase($database, $logError);
|
||||
break;
|
||||
default:
|
||||
Console::error("Unsupported usage aggregation type");
|
||||
}
|
||||
$interval = (int) App::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '30'); // 30 seconds (by default)
|
||||
$usage = new TimeSeries($database, $influxDB, $logError);
|
||||
|
||||
Console::loop(function () use ($interval, $usage) {
|
||||
$now = date('d-m-Y H:i:s', time());
|
||||
Console::info("[{$now}] Aggregating Timeseries Usage data every {$interval} seconds");
|
||||
$loopStart = microtime(true);
|
||||
|
||||
$usage->collect();
|
||||
|
||||
$loopTook = microtime(true) - $loopStart;
|
||||
$now = date('d-m-Y H:i:s', time());
|
||||
Console::info("[{$now}] Aggregation took {$loopTook} seconds");
|
||||
}, $interval);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -550,12 +550,10 @@ services:
|
|||
- _APP_MAINTENANCE_RETENTION_ABUSE
|
||||
- _APP_MAINTENANCE_RETENTION_AUDIT
|
||||
|
||||
appwrite-usage-timeseries:
|
||||
appwrite-usage:
|
||||
image: <?php echo $organization; ?>/<?php echo $image; ?>:<?php echo $version."\n"; ?>
|
||||
entrypoint:
|
||||
- usage
|
||||
- --type=timeseries
|
||||
container_name: appwrite-usage-timeseries
|
||||
entrypoint: usage
|
||||
container_name: appwrite-usage
|
||||
<<: *x-logging
|
||||
restart: unless-stopped
|
||||
networks:
|
||||
|
|
@ -573,40 +571,7 @@ services:
|
|||
- _APP_DB_PASS
|
||||
- _APP_INFLUXDB_HOST
|
||||
- _APP_INFLUXDB_PORT
|
||||
- _APP_USAGE_TIMESERIES_INTERVAL
|
||||
- _APP_USAGE_DATABASE_INTERVAL
|
||||
- _APP_REDIS_HOST
|
||||
- _APP_REDIS_PORT
|
||||
- _APP_REDIS_USER
|
||||
- _APP_REDIS_PASS
|
||||
- _APP_LOGGING_PROVIDER
|
||||
- _APP_LOGGING_CONFIG
|
||||
|
||||
appwrite-usage-database:
|
||||
image: <?php echo $organization; ?>/<?php echo $image; ?>:<?php echo $version."\n"; ?>
|
||||
entrypoint:
|
||||
- usage
|
||||
- --type=database
|
||||
container_name: appwrite-usage-database
|
||||
<<: *x-logging
|
||||
restart: unless-stopped
|
||||
networks:
|
||||
- appwrite
|
||||
depends_on:
|
||||
- influxdb
|
||||
- mariadb
|
||||
environment:
|
||||
- _APP_ENV
|
||||
- _APP_OPENSSL_KEY_V1
|
||||
- _APP_DB_HOST
|
||||
- _APP_DB_PORT
|
||||
- _APP_DB_SCHEMA
|
||||
- _APP_DB_USER
|
||||
- _APP_DB_PASS
|
||||
- _APP_INFLUXDB_HOST
|
||||
- _APP_INFLUXDB_PORT
|
||||
- _APP_USAGE_TIMESERIES_INTERVAL
|
||||
- _APP_USAGE_DATABASE_INTERVAL
|
||||
- _APP_USAGE_AGGREGATION_INTERVAL
|
||||
- _APP_REDIS_HOST
|
||||
- _APP_REDIS_PORT
|
||||
- _APP_REDIS_USER
|
||||
|
|
|
|||
|
|
@ -567,12 +567,10 @@ services:
|
|||
- _APP_MAINTENANCE_RETENTION_ABUSE
|
||||
- _APP_MAINTENANCE_RETENTION_AUDIT
|
||||
|
||||
appwrite-usage-timeseries:
|
||||
entrypoint:
|
||||
- usage
|
||||
- --type=timeseries
|
||||
appwrite-usage:
|
||||
entrypoint: usage
|
||||
<<: *x-logging
|
||||
container_name: appwrite-usage-timeseries
|
||||
container_name: appwrite-usage
|
||||
image: appwrite-dev
|
||||
networks:
|
||||
- appwrite
|
||||
|
|
@ -593,52 +591,13 @@ services:
|
|||
- _APP_DB_PASS
|
||||
- _APP_INFLUXDB_HOST
|
||||
- _APP_INFLUXDB_PORT
|
||||
- _APP_USAGE_TIMESERIES_INTERVAL
|
||||
- _APP_USAGE_DATABASE_INTERVAL
|
||||
- _APP_USAGE_AGGREGATION_INTERVAL
|
||||
- _APP_REDIS_HOST
|
||||
- _APP_REDIS_PORT
|
||||
- _APP_REDIS_USER
|
||||
- _APP_REDIS_PASS
|
||||
- _APP_LOGGING_PROVIDER
|
||||
- _APP_LOGGING_CONFIG
|
||||
|
||||
# appwrite-usage-database:
|
||||
# entrypoint:
|
||||
# - usage
|
||||
# - --type=database
|
||||
# <<: *x-logging
|
||||
# container_name: appwrite-usage-database
|
||||
# build:
|
||||
# context: .
|
||||
# args:
|
||||
# - DEBUG=false
|
||||
# networks:
|
||||
# - appwrite
|
||||
# volumes:
|
||||
# - ./app:/usr/src/code/app
|
||||
# - ./src:/usr/src/code/src
|
||||
# - ./dev:/usr/local/dev
|
||||
# depends_on:
|
||||
# - influxdb
|
||||
# - mariadb
|
||||
# environment:
|
||||
# - _APP_ENV
|
||||
# - _APP_OPENSSL_KEY_V1
|
||||
# - _APP_DB_HOST
|
||||
# - _APP_DB_PORT
|
||||
# - _APP_DB_SCHEMA
|
||||
# - _APP_DB_USER
|
||||
# - _APP_DB_PASS
|
||||
# - _APP_INFLUXDB_HOST
|
||||
# - _APP_INFLUXDB_PORT
|
||||
# - _APP_USAGE_TIMESERIES_INTERVAL
|
||||
# - _APP_USAGE_DATABASE_INTERVAL
|
||||
# - _APP_REDIS_HOST
|
||||
# - _APP_REDIS_PORT
|
||||
# - _APP_REDIS_USER
|
||||
# - _APP_REDIS_PASS
|
||||
# - _APP_LOGGING_PROVIDER
|
||||
# - _APP_LOGGING_CONFIG
|
||||
|
||||
appwrite-schedule:
|
||||
entrypoint: schedule
|
||||
|
|
|
|||
|
|
@ -1,231 +0,0 @@
|
|||
<?php
|
||||
|
||||
namespace Appwrite\Usage\Calculators;
|
||||
|
||||
use DateTime;
|
||||
use Utopia\Database\Database as UtopiaDatabase;
|
||||
use Utopia\Database\Document;
|
||||
use Utopia\Database\Query;
|
||||
|
||||
class Aggregator extends Database
|
||||
{
|
||||
protected function aggregateDatabaseMetrics(string $projectId): void
|
||||
{
|
||||
$this->database->setNamespace('_' . $projectId);
|
||||
|
||||
$databasesGeneralMetrics = [
|
||||
'databases.$all.requests.create',
|
||||
'databases.$all.requests.read',
|
||||
'databases.$all.requests.update',
|
||||
'databases.$all.requests.delete',
|
||||
'collections.$all.requests.create',
|
||||
'collections.$all.requests.read',
|
||||
'collections.$all.requests.update',
|
||||
'collections.$all.requests.delete',
|
||||
'documents.$all.requests.create',
|
||||
'documents.$all.requests.read',
|
||||
'documents.$all.requests.update',
|
||||
'documents.$all.requests.delete'
|
||||
];
|
||||
|
||||
foreach ($databasesGeneralMetrics as $metric) {
|
||||
$this->aggregateDailyMetric($projectId, $metric);
|
||||
$this->aggregateMonthlyMetric($projectId, $metric);
|
||||
}
|
||||
|
||||
$databasesDatabaseMetrics = [
|
||||
'collections.databaseId.requests.create',
|
||||
'collections.databaseId.requests.read',
|
||||
'collections.databaseId.requests.update',
|
||||
'collections.databaseId.requests.delete',
|
||||
'documents.databaseId.requests.create',
|
||||
'documents.databaseId.requests.read',
|
||||
'documents.databaseId.requests.update',
|
||||
'documents.databaseId.requests.delete',
|
||||
];
|
||||
|
||||
$this->foreachDocument($projectId, 'databases', [], function (Document $database) use ($databasesDatabaseMetrics, $projectId) {
|
||||
$databaseId = $database->getId();
|
||||
foreach ($databasesDatabaseMetrics as $metric) {
|
||||
$metric = str_replace('databaseId', $databaseId, $metric);
|
||||
$this->aggregateDailyMetric($projectId, $metric);
|
||||
$this->aggregateMonthlyMetric($projectId, $metric);
|
||||
}
|
||||
|
||||
$databasesCollectionMetrics = [
|
||||
'documents.' . $databaseId . '/collectionId.requests.create',
|
||||
'documents.' . $databaseId . '/collectionId.requests.read',
|
||||
'documents.' . $databaseId . '/collectionId.requests.update',
|
||||
'documents.' . $databaseId . '/collectionId.requests.delete',
|
||||
];
|
||||
|
||||
$this->foreachDocument($projectId, 'database_' . $database->getInternalId(), [], function (Document $collection) use ($databasesCollectionMetrics, $projectId) {
|
||||
$collectionId = $collection->getId();
|
||||
foreach ($databasesCollectionMetrics as $metric) {
|
||||
$metric = str_replace('collectionId', $collectionId, $metric);
|
||||
$this->aggregateDailyMetric($projectId, $metric);
|
||||
$this->aggregateMonthlyMetric($projectId, $metric);
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
protected function aggregateStorageMetrics(string $projectId): void
|
||||
{
|
||||
$this->database->setNamespace('_' . $projectId);
|
||||
|
||||
$storageGeneralMetrics = [
|
||||
'buckets.$all.requests.create',
|
||||
'buckets.$all.requests.read',
|
||||
'buckets.$all.requests.update',
|
||||
'buckets.$all.requests.delete',
|
||||
'files.$all.requests.create',
|
||||
'files.$all.requests.read',
|
||||
'files.$all.requests.update',
|
||||
'files.$all.requests.delete',
|
||||
];
|
||||
|
||||
foreach ($storageGeneralMetrics as $metric) {
|
||||
$this->aggregateDailyMetric($projectId, $metric);
|
||||
$this->aggregateMonthlyMetric($projectId, $metric);
|
||||
}
|
||||
|
||||
$storageBucketMetrics = [
|
||||
'files.bucketId.requests.create',
|
||||
'files.bucketId.requests.read',
|
||||
'files.bucketId.requests.update',
|
||||
'files.bucketId.requests.delete',
|
||||
];
|
||||
|
||||
$this->foreachDocument($projectId, 'buckets', [], function (Document $bucket) use ($storageBucketMetrics, $projectId) {
|
||||
$bucketId = $bucket->getId();
|
||||
foreach ($storageBucketMetrics as $metric) {
|
||||
$metric = str_replace('bucketId', $bucketId, $metric);
|
||||
$this->aggregateDailyMetric($projectId, $metric);
|
||||
$this->aggregateMonthlyMetric($projectId, $metric);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
protected function aggregateFunctionMetrics(string $projectId): void
|
||||
{
|
||||
$this->database->setNamespace('_' . $projectId);
|
||||
|
||||
$functionsGeneralMetrics = [
|
||||
'project.$all.compute.total',
|
||||
'project.$all.compute.time',
|
||||
'executions.$all.compute.total',
|
||||
'executions.$all.compute.success',
|
||||
'executions.$all.compute.failure',
|
||||
'executions.$all.compute.time',
|
||||
'builds.$all.compute.total',
|
||||
'builds.$all.compute.success',
|
||||
'builds.$all.compute.failure',
|
||||
'builds.$all.compute.time',
|
||||
];
|
||||
|
||||
foreach ($functionsGeneralMetrics as $metric) {
|
||||
$this->aggregateDailyMetric($projectId, $metric);
|
||||
$this->aggregateMonthlyMetric($projectId, $metric);
|
||||
}
|
||||
|
||||
$functionMetrics = [
|
||||
'executions.functionId.compute.total',
|
||||
'executions.functionId.compute.success',
|
||||
'executions.functionId.compute.failure',
|
||||
'executions.functionId.compute.time',
|
||||
'builds.functionId.compute.total',
|
||||
'builds.functionId.compute.success',
|
||||
'builds.functionId.compute.failure',
|
||||
'builds.functionId.compute.time',
|
||||
];
|
||||
|
||||
$this->foreachDocument($projectId, 'functions', [], function (Document $function) use ($functionMetrics, $projectId) {
|
||||
$functionId = $function->getId();
|
||||
foreach ($functionMetrics as $metric) {
|
||||
$metric = str_replace('functionId', $functionId, $metric);
|
||||
$this->aggregateDailyMetric($projectId, $metric);
|
||||
$this->aggregateMonthlyMetric($projectId, $metric);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
protected function aggregateUsersMetrics(string $projectId): void
|
||||
{
|
||||
$metrics = [
|
||||
'users.$all.requests.create',
|
||||
'users.$all.requests.read',
|
||||
'users.$all.requests.update',
|
||||
'users.$all.requests.delete',
|
||||
'sessions.$all.requests.create',
|
||||
'sessions.$all.requests.delete'
|
||||
];
|
||||
|
||||
foreach ($metrics as $metric) {
|
||||
$this->aggregateDailyMetric($projectId, $metric);
|
||||
$this->aggregateMonthlyMetric($projectId, $metric);
|
||||
}
|
||||
}
|
||||
|
||||
protected function aggregateGeneralMetrics(string $projectId): void
|
||||
{
|
||||
$this->aggregateDailyMetric($projectId, 'project.$all.network.requests');
|
||||
$this->aggregateDailyMetric($projectId, 'project.$all.network.bandwidth');
|
||||
$this->aggregateDailyMetric($projectId, 'project.$all.network.inbound');
|
||||
$this->aggregateDailyMetric($projectId, 'project.$all.network.outbound');
|
||||
$this->aggregateMonthlyMetric($projectId, 'project.$all.network.requests');
|
||||
$this->aggregateMonthlyMetric($projectId, 'project.$all.network.bandwidth');
|
||||
$this->aggregateMonthlyMetric($projectId, 'project.$all.network.inbound');
|
||||
$this->aggregateMonthlyMetric($projectId, 'project.$all.network.outbound');
|
||||
}
|
||||
|
||||
protected function aggregateDailyMetric(string $projectId, string $metric): void
|
||||
{
|
||||
$beginOfDay = DateTime::createFromFormat('Y-m-d\TH:i:s.v', \date('Y-m-d\T00:00:00.000'))->format(DateTime::RFC3339);
|
||||
$endOfDay = DateTime::createFromFormat('Y-m-d\TH:i:s.v', \date('Y-m-d\T23:59:59.999'))->format(DateTime::RFC3339);
|
||||
|
||||
$this->database->setNamespace('_' . $projectId);
|
||||
$value = (int) $this->database->sum('stats', 'value', [
|
||||
Query::equal('metric', [$metric]),
|
||||
Query::equal('period', ['1h']),
|
||||
Query::greaterThanEqual('time', $beginOfDay),
|
||||
Query::lessThanEqual('time', $endOfDay),
|
||||
]);
|
||||
$this->createOrUpdateMetric($projectId, $metric, '1d', $beginOfDay, $value);
|
||||
}
|
||||
|
||||
protected function aggregateMonthlyMetric(string $projectId, string $metric): void
|
||||
{
|
||||
$beginOfMonth = DateTime::createFromFormat('Y-m-d\TH:i:s.v', \date('Y-m-01\T00:00:00.000'))->format(DateTime::RFC3339);
|
||||
$endOfMonth = DateTime::createFromFormat('Y-m-d\TH:i:s.v', \date('Y-m-t\T23:59:59.999'))->format(DateTime::RFC3339);
|
||||
$this->database->setNamespace('_' . $projectId);
|
||||
$value = (int) $this->database->sum('stats', 'value', [
|
||||
Query::equal('metric', [$metric]),
|
||||
Query::equal('period', ['1d']),
|
||||
Query::greaterThanEqual('time', $beginOfMonth),
|
||||
Query::lessThanEqual('time', $endOfMonth),
|
||||
]);
|
||||
$this->createOrUpdateMetric($projectId, $metric, '1mo', $beginOfMonth, $value);
|
||||
}
|
||||
|
||||
/**
|
||||
* Collect Stats
|
||||
* Collect all database related stats
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
public function collect(): void
|
||||
{
|
||||
$this->foreachDocument('console', 'projects', [], function (Document $project) {
|
||||
$projectId = $project->getInternalId();
|
||||
|
||||
// Aggregate new metrics from already collected usage metrics
|
||||
// for lower time period (1day and 1 month metric from 30 minute metrics)
|
||||
$this->aggregateGeneralMetrics($projectId);
|
||||
$this->aggregateFunctionMetrics($projectId);
|
||||
$this->aggregateDatabaseMetrics($projectId);
|
||||
$this->aggregateStorageMetrics($projectId);
|
||||
$this->aggregateUsersMetrics($projectId);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
|
@ -1,360 +0,0 @@
|
|||
<?php
|
||||
|
||||
namespace Appwrite\Usage\Calculators;
|
||||
|
||||
use Exception;
|
||||
use Appwrite\Usage\Calculator;
|
||||
use DateTime;
|
||||
use Utopia\Database\Database as UtopiaDatabase;
|
||||
use Utopia\Database\Document;
|
||||
use Utopia\Database\Exception\Authorization;
|
||||
use Utopia\Database\Exception\Structure;
|
||||
use Utopia\Database\Query;
|
||||
|
||||
class Database extends Calculator
|
||||
{
|
||||
protected array $periods = [
|
||||
[
|
||||
'key' => '1h',
|
||||
'multiplier' => 3600,
|
||||
],
|
||||
[
|
||||
'key' => '1d',
|
||||
'multiplier' => 86400,
|
||||
],
|
||||
];
|
||||
|
||||
public function __construct(UtopiaDatabase $database, callable $errorHandler = null)
|
||||
{
|
||||
$this->database = $database;
|
||||
$this->errorHandler = $errorHandler;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create Per Period Metric
|
||||
*
|
||||
* Create given metric for each defined period
|
||||
*
|
||||
* @param string $projectId
|
||||
* @param string $metric
|
||||
* @param int $value
|
||||
* @param bool $monthly
|
||||
* @return void
|
||||
* @throws Authorization
|
||||
* @throws Structure
|
||||
*/
|
||||
protected function createPerPeriodMetric(string $projectId, string $metric, int $value, bool $monthly = false): void
|
||||
{
|
||||
foreach ($this->periods as $options) {
|
||||
$period = $options['key'];
|
||||
$date = new \DateTime();
|
||||
if ($period === '1h') {
|
||||
$minutes = $date->format('i') >= '30' ? "30" : "00";
|
||||
$time = $date->format('Y-m-d H:' . $minutes . ':00');
|
||||
} elseif ($period === '1d') {
|
||||
$time = $date->format('Y-m-d 00:00:00');
|
||||
} else {
|
||||
throw new Exception("Period type not found", 500);
|
||||
}
|
||||
$this->createOrUpdateMetric($projectId, $metric, $period, $time, $value);
|
||||
}
|
||||
|
||||
// Required for billing
|
||||
if ($monthly) {
|
||||
$time = DateTime::createFromFormat('Y-m-d\TH:i:s.v', \date('Y-m-01\T00:00:00.000'))->format(DateTime::RFC3339);
|
||||
$this->createOrUpdateMetric($projectId, $metric, '1mo', $time, $value);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create or Update Metric
|
||||
*
|
||||
* Create or update each metric in the stats collection for the given project
|
||||
*
|
||||
* @param string $projectId
|
||||
* @param string $metric
|
||||
* @param string $period
|
||||
* @param string $time
|
||||
* @param int $value
|
||||
*
|
||||
* @return void
|
||||
* @throws Authorization
|
||||
* @throws Structure
|
||||
*/
|
||||
protected function createOrUpdateMetric(string $projectId, string $metric, string $period, string $time, int $value): void
|
||||
{
|
||||
$id = \md5("{$time}_{$period}_{$metric}");
|
||||
$this->database->setNamespace('_' . $projectId);
|
||||
|
||||
try {
|
||||
$document = $this->database->getDocument('stats', $id);
|
||||
if ($document->isEmpty()) {
|
||||
$this->database->createDocument('stats', new Document([
|
||||
'$id' => $id,
|
||||
'period' => $period,
|
||||
'time' => $time,
|
||||
'metric' => $metric,
|
||||
'value' => $value,
|
||||
'type' => 2, // these are cumulative metrics
|
||||
]));
|
||||
} else {
|
||||
$this->database->updateDocument(
|
||||
'stats',
|
||||
$document->getId(),
|
||||
$document->setAttribute('value', $value)
|
||||
);
|
||||
}
|
||||
} catch (\Exception$e) { // if projects are deleted this might fail
|
||||
if (is_callable($this->errorHandler)) {
|
||||
call_user_func($this->errorHandler, $e, "sync_project_{$projectId}_metric_{$metric}");
|
||||
} else {
|
||||
throw $e;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Foreach Document
|
||||
*
|
||||
* Call provided callback for each document in the collection
|
||||
*
|
||||
* @param string $projectId
|
||||
* @param string $collection
|
||||
* @param array $queries
|
||||
* @param callable $callback
|
||||
*
|
||||
* @return void
|
||||
* @throws Exception
|
||||
*/
|
||||
protected function foreachDocument(string $projectId, string $collection, array $queries, callable $callback): void
|
||||
{
|
||||
$limit = 50;
|
||||
$results = [];
|
||||
$sum = $limit;
|
||||
$latestDocument = null;
|
||||
$this->database->setNamespace('_' . $projectId);
|
||||
|
||||
while ($sum === $limit) {
|
||||
try {
|
||||
$paginationQueries = [Query::limit($limit)];
|
||||
if ($latestDocument !== null) {
|
||||
$paginationQueries[] = Query::cursorAfter($latestDocument);
|
||||
}
|
||||
$results = $this->database->find($collection, \array_merge($paginationQueries, $queries));
|
||||
} catch (\Exception $e) {
|
||||
if (is_callable($this->errorHandler)) {
|
||||
call_user_func($this->errorHandler, $e, "fetch_documents_project_{$projectId}_collection_{$collection}");
|
||||
return;
|
||||
} else {
|
||||
throw $e;
|
||||
}
|
||||
}
|
||||
if (empty($results)) {
|
||||
return;
|
||||
}
|
||||
|
||||
$sum = count($results);
|
||||
|
||||
foreach ($results as $document) {
|
||||
if (is_callable($callback)) {
|
||||
$callback($document);
|
||||
}
|
||||
}
|
||||
$latestDocument = $results[array_key_last($results)];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Sum
|
||||
*
|
||||
* Calculate sum of an attribute of documents in collection
|
||||
*
|
||||
* @param string $projectId
|
||||
* @param string $collection
|
||||
* @param string $attribute
|
||||
* @param string|null $metric
|
||||
* @param int $multiplier
|
||||
* @return int
|
||||
* @throws Exception
|
||||
*/
|
||||
private function sum(string $projectId, string $collection, string $attribute, string $metric = null, int $multiplier = 1): int
|
||||
{
|
||||
$this->database->setNamespace('_' . $projectId);
|
||||
|
||||
try {
|
||||
$sum = $this->database->sum($collection, $attribute);
|
||||
$sum = (int) ($sum * $multiplier);
|
||||
|
||||
if (!is_null($metric)) {
|
||||
$this->createPerPeriodMetric($projectId, $metric, $sum);
|
||||
}
|
||||
return $sum;
|
||||
} catch (Exception $e) {
|
||||
if (is_callable($this->errorHandler)) {
|
||||
call_user_func($this->errorHandler, $e, "fetch_sum_project_{$projectId}_collection_{$collection}");
|
||||
} else {
|
||||
throw $e;
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Count
|
||||
*
|
||||
* Count number of documents in collection
|
||||
*
|
||||
* @param string $projectId
|
||||
* @param string $collection
|
||||
* @param ?string $metric
|
||||
*
|
||||
* @return int
|
||||
* @throws Exception
|
||||
*/
|
||||
private function count(string $projectId, string $collection, ?string $metric = null): int
|
||||
{
|
||||
$this->database->setNamespace('_' . $projectId);
|
||||
|
||||
try {
|
||||
$count = $this->database->count($collection);
|
||||
if (!is_null($metric)) {
|
||||
$this->createPerPeriodMetric($projectId, (string) $metric, $count);
|
||||
}
|
||||
return $count;
|
||||
} catch (Exception $e) {
|
||||
if (is_callable($this->errorHandler)) {
|
||||
call_user_func($this->errorHandler, $e, "fetch_count_project_{$projectId}_collection_{$collection}");
|
||||
} else {
|
||||
throw $e;
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Deployments Total
|
||||
*
|
||||
* Total sum of storage used by deployments
|
||||
*
|
||||
* @param string $projectId
|
||||
*
|
||||
* @return int
|
||||
* @throws Exception
|
||||
*/
|
||||
private function deploymentsTotal(string $projectId): int
|
||||
{
|
||||
return $this->sum($projectId, 'deployments', 'size', 'deployments.$all.storage.size');
|
||||
}
|
||||
|
||||
/**
|
||||
* Users Stats
|
||||
*
|
||||
* Metric: users.count
|
||||
*
|
||||
* @param string $projectId
|
||||
*
|
||||
* @return void
|
||||
* @throws Exception
|
||||
*/
|
||||
private function usersStats(string $projectId): void
|
||||
{
|
||||
$this->count($projectId, 'users', 'users.$all.count.total');
|
||||
}
|
||||
|
||||
/**
|
||||
* Storage Stats
|
||||
*
|
||||
* Metrics: buckets.$all.count.total, files.$all.count.total, files.bucketId,count.total,
|
||||
* files.$all.storage.size, files.bucketId.storage.size, project.$all.storage.size
|
||||
*
|
||||
* @param string $projectId
|
||||
*
|
||||
* @return void
|
||||
* @throws Authorization
|
||||
* @throws Structure
|
||||
*/
|
||||
private function storageStats(string $projectId): void
|
||||
{
|
||||
$projectFilesTotal = 0;
|
||||
$projectFilesCount = 0;
|
||||
|
||||
$metric = 'buckets.$all.count.total';
|
||||
$this->count($projectId, 'buckets', $metric);
|
||||
|
||||
$this->foreachDocument($projectId, 'buckets', [], function ($bucket) use (&$projectFilesCount, &$projectFilesTotal, $projectId,) {
|
||||
$metric = "files.{$bucket->getId()}.count.total";
|
||||
$count = $this->count($projectId, 'bucket_' . $bucket->getInternalId(), $metric);
|
||||
$projectFilesCount += $count;
|
||||
|
||||
$metric = "files.{$bucket->getId()}.storage.size";
|
||||
$sum = $this->sum($projectId, 'bucket_' . $bucket->getInternalId(), 'sizeOriginal', $metric);
|
||||
$projectFilesTotal += $sum;
|
||||
});
|
||||
|
||||
$this->createPerPeriodMetric($projectId, 'files.$all.count.total', $projectFilesCount);
|
||||
$this->createPerPeriodMetric($projectId, 'files.$all.storage.size', $projectFilesTotal);
|
||||
|
||||
$deploymentsTotal = $this->deploymentsTotal($projectId);
|
||||
$this->createPerPeriodMetric($projectId, 'project.$all.storage.size', $projectFilesTotal + $deploymentsTotal);
|
||||
}
|
||||
|
||||
/**
|
||||
* Database Stats
|
||||
*
|
||||
* Collect all database stats
|
||||
* Metrics: databases.$all.count.total, collections.$all.count.total, collections.databaseId.count.total,
|
||||
* documents.$all.count.all, documents.databaseId.count.total, documents.databaseId/collectionId.count.total
|
||||
*
|
||||
* @param string $projectId
|
||||
*
|
||||
* @return void
|
||||
* @throws Authorization
|
||||
* @throws Structure
|
||||
*/
|
||||
private function databaseStats(string $projectId): void
|
||||
{
|
||||
$projectDocumentsCount = 0;
|
||||
$projectCollectionsCount = 0;
|
||||
|
||||
$this->count($projectId, 'databases', 'databases.$all.count.total');
|
||||
|
||||
$this->foreachDocument($projectId, 'databases', [], function ($database) use (&$projectDocumentsCount, &$projectCollectionsCount, $projectId) {
|
||||
$metric = "collections.{$database->getId()}.count.total";
|
||||
$count = $this->count($projectId, 'database_' . $database->getInternalId(), $metric);
|
||||
$projectCollectionsCount += $count;
|
||||
$databaseDocumentsCount = 0;
|
||||
|
||||
$this->foreachDocument($projectId, 'database_' . $database->getInternalId(), [], function ($collection) use (&$projectDocumentsCount, &$databaseDocumentsCount, $projectId, $database) {
|
||||
$metric = "documents.{$database->getId()}/{$collection->getId()}.count.total";
|
||||
|
||||
$count = $this->count($projectId, 'database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $metric);
|
||||
$projectDocumentsCount += $count;
|
||||
$databaseDocumentsCount += $count;
|
||||
});
|
||||
|
||||
$this->createPerPeriodMetric($projectId, "documents.{$database->getId()}.count.total", $databaseDocumentsCount);
|
||||
});
|
||||
|
||||
$this->createPerPeriodMetric($projectId, 'collections.$all.count.total', $projectCollectionsCount);
|
||||
$this->createPerPeriodMetric($projectId, 'documents.$all.count.total', $projectDocumentsCount);
|
||||
}
|
||||
|
||||
/**
|
||||
* Collect Stats
|
||||
*
|
||||
* Collect all database related stats
|
||||
*
|
||||
* @return void
|
||||
* @throws Exception
|
||||
*/
|
||||
public function collect(): void
|
||||
{
|
||||
$this->foreachDocument('console', 'projects', [], function (Document $project) {
|
||||
$projectId = $project->getInternalId();
|
||||
|
||||
$this->usersStats($projectId);
|
||||
$this->databaseStats($projectId);
|
||||
$this->storageStats($projectId);
|
||||
});
|
||||
}
|
||||
}
|
||||
Loading…
Reference in a new issue