2021-08-10 20:44:31 +12:00
< ? php
2021-08-11 22:15:00 +12:00
global $cli , $register ;
2021-08-10 20:44:31 +12:00
use Utopia\App ;
2021-08-13 20:39:46 +12:00
use Utopia\Cache\Adapter\Redis ;
2021-08-11 22:15:00 +12:00
use Utopia\Cache\Cache ;
2021-08-10 20:44:31 +12:00
use Utopia\CLI\Console ;
2021-08-11 22:15:00 +12:00
use Utopia\Database\Adapter\MariaDB ;
use Utopia\Database\Database ;
use Utopia\Database\Document ;
use Utopia\Database\Validator\Authorization ;
2021-08-10 20:44:31 +12:00
2021-08-19 18:54:32 +12:00
/**
* Metrics We collect
2021-08-19 20:01:22 +12:00
*
2021-08-30 19:19:29 +12:00
* General
2021-08-30 21:35:32 +12:00
*
2021-08-19 18:54:32 +12:00
* requests
* network
* executions
2021-08-30 21:35:32 +12:00
*
2021-08-30 19:19:29 +12:00
* Database
2021-08-30 21:35:32 +12:00
*
2021-08-19 18:54:32 +12:00
* database . collections . create
* database . collections . read
* database . collections . update
* database . collections . delete
* database . documents . create
* database . documents . read
* database . documents . update
* database . documents . delete
* database . collections . { collectionId } . documents . create
* database . collections . { collectionId } . documents . read
* database . collections . { collectionId } . documents . update
* database . collections . { collectionId } . documents . delete
2021-08-30 21:35:32 +12:00
*
2021-08-30 19:19:29 +12:00
* Storage
2022-05-24 02:54:50 +12:00
*
2021-09-10 20:49:16 +12:00
* storage . buckets . create
* storage . buckets . read
* storage . buckets . update
* storage . buckets . delete
2021-11-09 18:50:03 +13:00
* storage . files . create
* storage . files . read
* storage . files . update
* storage . files . delete
2021-08-19 18:54:32 +12:00
* storage . buckets . { bucketId } . files . create
* storage . buckets . { bucketId } . files . read
* storage . buckets . { bucketId } . files . update
* storage . buckets . { bucketId } . files . delete
2021-08-30 21:35:32 +12:00
*
2021-08-30 19:19:29 +12:00
* Users
2021-08-30 21:35:32 +12:00
*
2021-08-19 18:54:32 +12:00
* users . create
* users . read
* users . update
* users . delete
* users . sessions . create
2021-08-24 20:36:46 +12:00
* users . sessions . { provider } . create
2021-08-19 18:54:32 +12:00
* users . sessions . delete
2021-08-19 20:01:22 +12:00
*
2021-08-20 17:55:23 +12:00
* Functions
*
* functions . { functionId } . executions
* functions . { functionId } . failures
* functions . { functionId } . compute
*
2021-08-19 18:54:32 +12:00
* Counters
2021-08-19 20:01:22 +12:00
*
2021-08-19 18:54:32 +12:00
* users . count
2021-09-10 20:49:16 +12:00
* storage . buckets . count
2021-08-19 20:01:22 +12:00
* storage . files . count
2021-11-09 18:50:03 +13:00
* storage . buckets . { bucketId } . files . count
2021-08-19 20:01:22 +12:00
* database . collections . count
* database . documents . count
* database . collections . { collectionId } . documents . count
*
2021-08-19 20:14:23 +12:00
* Totals
*
* storage . total
*
2021-08-19 18:54:32 +12:00
*/
2021-08-10 20:44:31 +12:00
$cli
-> task ( 'usage' )
-> desc ( 'Schedules syncing data from influxdb to Appwrite console db' )
2021-08-11 22:15:00 +12:00
-> action ( function () use ( $register ) {
2021-08-17 00:31:49 +12:00
Console :: title ( 'Usage Aggregation V1' );
Console :: success ( APP_NAME . ' usage aggregation process v1 has started' );
2021-08-10 20:44:31 +12:00
2021-08-30 19:19:29 +12:00
$interval = ( int ) App :: getEnv ( '_APP_USAGE_AGGREGATION_INTERVAL' , '30' ); // 30 seconds (by default)
2021-08-16 18:58:34 +12:00
$periods = [
[
'key' => '30m' ,
'startTime' => '-24 hours' ,
],
[
'key' => '1d' ,
'startTime' => '-90 days' ,
],
];
2021-08-17 00:31:49 +12:00
// all the metrics that we are collecting at the moment
2021-08-16 18:58:34 +12:00
$globalMetrics = [
'requests' => [
'table' => 'appwrite_usage_requests_all' ,
],
'network' => [
'table' => 'appwrite_usage_network_all' ,
],
'executions' => [
'table' => 'appwrite_usage_executions_all' ,
],
'database.collections.create' => [
'table' => 'appwrite_usage_database_collections_create' ,
],
'database.collections.read' => [
'table' => 'appwrite_usage_database_collections_read' ,
],
'database.collections.update' => [
'table' => 'appwrite_usage_database_collections_update' ,
],
'database.collections.delete' => [
'table' => 'appwrite_usage_database_collections_delete' ,
],
'database.documents.create' => [
'table' => 'appwrite_usage_database_documents_create' ,
],
'database.documents.read' => [
'table' => 'appwrite_usage_database_documents_read' ,
],
'database.documents.update' => [
'table' => 'appwrite_usage_database_documents_update' ,
],
'database.documents.delete' => [
'table' => 'appwrite_usage_database_documents_delete' ,
],
2021-08-17 00:22:54 +12:00
'database.collections.collectionId.documents.create' => [
2021-08-16 18:58:34 +12:00
'table' => 'appwrite_usage_database_documents_create' ,
'groupBy' => 'collectionId' ,
],
2021-08-17 00:22:54 +12:00
'database.collections.collectionId.documents.read' => [
2021-08-16 18:58:34 +12:00
'table' => 'appwrite_usage_database_documents_read' ,
'groupBy' => 'collectionId' ,
],
2021-08-17 00:22:54 +12:00
'database.collections.collectionId.documents.update' => [
2021-08-16 18:58:34 +12:00
'table' => 'appwrite_usage_database_documents_update' ,
'groupBy' => 'collectionId' ,
],
2021-08-17 00:22:54 +12:00
'database.collections.collectionId.documents.delete' => [
2021-08-16 18:58:34 +12:00
'table' => 'appwrite_usage_database_documents_delete' ,
'groupBy' => 'collectionId' ,
],
2021-09-10 20:49:16 +12:00
'storage.buckets.create' => [
'table' => 'appwrite_usage_storage_buckets_create' ,
],
'storage.buckets.read' => [
'table' => 'appwrite_usage_storage_buckets_read' ,
],
'storage.buckets.update' => [
'table' => 'appwrite_usage_storage_buckets_update' ,
],
'storage.buckets.delete' => [
'table' => 'appwrite_usage_storage_buckets_delete' ,
],
2021-11-09 18:50:03 +13:00
'storage.files.create' => [
'table' => 'appwrite_usage_storage_files_create' ,
],
'storage.files.read' => [
'table' => 'appwrite_usage_storage_files_read' ,
],
'storage.files.update' => [
'table' => 'appwrite_usage_storage_files_update' ,
],
'storage.files.delete' => [
'table' => 'appwrite_usage_storage_files_delete' ,
],
2021-08-16 19:25:20 +12:00
'storage.buckets.bucketId.files.create' => [
'table' => 'appwrite_usage_storage_files_create' ,
'groupBy' => 'bucketId' ,
],
'storage.buckets.bucketId.files.read' => [
'table' => 'appwrite_usage_storage_files_read' ,
'groupBy' => 'bucketId' ,
],
'storage.buckets.bucketId.files.update' => [
'table' => 'appwrite_usage_storage_files_update' ,
'groupBy' => 'bucketId' ,
],
'storage.buckets.bucketId.files.delete' => [
'table' => 'appwrite_usage_storage_files_delete' ,
'groupBy' => 'bucketId' ,
],
2021-08-16 20:53:34 +12:00
'users.create' => [
'table' => 'appwrite_usage_users_create' ,
],
'users.read' => [
'table' => 'appwrite_usage_users_read' ,
],
'users.update' => [
'table' => 'appwrite_usage_users_update' ,
],
'users.delete' => [
'table' => 'appwrite_usage_users_delete' ,
],
'users.sessions.create' => [
'table' => 'appwrite_usage_users_sessions_create' ,
2021-08-24 20:36:46 +12:00
],
'users.sessions.provider.create' => [
'table' => 'appwrite_usage_users_sessions_create' ,
2021-08-16 20:53:34 +12:00
'groupBy' => 'provider' ,
],
'users.sessions.delete' => [
'table' => 'appwrite_usage_users_sessions_delete' ,
],
2021-08-20 17:55:23 +12:00
'functions.functionId.executions' => [
'table' => 'appwrite_usage_executions_all' ,
'groupBy' => 'functionId' ,
],
'functions.functionId.compute' => [
'table' => 'appwrite_usage_executions_time' ,
'groupBy' => 'functionId' ,
],
'functions.functionId.failures' => [
'table' => 'appwrite_usage_executions_all' ,
'groupBy' => 'functionId' ,
'filters' => [
'functionStatus' => 'failed' ,
],
],
2021-08-16 18:58:34 +12:00
];
2021-08-30 19:19:29 +12:00
// TODO Maybe move this to the setResource method, and reuse in the http.php file
2021-08-13 21:45:46 +12:00
$attempts = 0 ;
$max = 10 ;
$sleep = 1 ;
2021-08-30 21:35:32 +12:00
2022-02-20 23:03:19 +13:00
$db = null ;
$redis = null ;
2021-08-16 18:58:34 +12:00
do { // connect to db
2021-08-13 21:45:46 +12:00
try {
$attempts ++ ;
$db = $register -> get ( 'db' );
$redis = $register -> get ( 'cache' );
break ; // leave the do-while if successful
2021-08-30 19:19:29 +12:00
} catch ( \Exception $e ) {
2021-08-13 21:45:46 +12:00
Console :: warning ( " Database not ready. Retrying connection ( { $attempts } )... " );
if ( $attempts >= $max ) {
throw new \Exception ( 'Failed to connect to database: ' . $e -> getMessage ());
}
sleep ( $sleep );
}
} while ( $attempts < $max );
2021-08-10 20:44:31 +12:00
2021-08-30 21:35:32 +12:00
// TODO use inject
2021-08-15 20:38:31 +12:00
$cacheAdapter = new Cache ( new Redis ( $redis ));
2021-08-13 21:45:46 +12:00
$dbForProject = new Database ( new MariaDB ( $db ), $cacheAdapter );
2021-08-17 18:03:27 +12:00
$dbForConsole = new Database ( new MariaDB ( $db ), $cacheAdapter );
2021-12-29 01:04:58 +13:00
$dbForProject -> setDefaultDatabase ( App :: getEnv ( '_APP_DB_SCHEMA' , 'appwrite' ));
$dbForConsole -> setDefaultDatabase ( App :: getEnv ( '_APP_DB_SCHEMA' , 'appwrite' ));
2022-02-08 01:25:15 +13:00
$dbForConsole -> setNamespace ( '_console' );
2021-08-10 20:44:31 +12:00
2021-08-16 21:02:35 +12:00
$latestTime = [];
2021-08-11 22:15:00 +12:00
Authorization :: disable ();
2021-08-10 20:44:31 +12:00
2021-08-17 18:03:27 +12:00
$iterations = 0 ;
Console :: loop ( function () use ( $interval , $register , $dbForProject , $dbForConsole , $globalMetrics , $periods , & $latestTime , & $iterations ) {
2021-08-16 21:02:35 +12:00
$now = date ( 'd-m-Y H:i:s' , time ());
Console :: info ( " [ { $now } ] Aggregating usage data every { $interval } seconds " );
2021-08-11 22:15:00 +12:00
2021-08-16 20:53:34 +12:00
$loopStart = microtime ( true );
2021-08-30 19:19:29 +12:00
/**
* Aggregate InfluxDB every 30 seconds
2021-09-10 22:23:49 +12:00
* @ var InfluxDB\Client $client
2021-08-30 19:19:29 +12:00
*/
2021-08-13 20:39:46 +12:00
$client = $register -> get ( 'influxdb' );
2022-02-20 23:03:19 +13:00
$attempts = 0 ;
$max = 10 ;
$sleep = 1 ;
2021-09-10 22:23:49 +12:00
2022-02-20 23:03:19 +13:00
do { // check if telegraf database is ready
try {
$attempts ++ ;
$database = $client -> selectDB ( 'telegraf' );
2022-05-24 02:54:50 +12:00
if ( in_array ( 'telegraf' , $client -> listDatabases ())) {
2022-02-20 23:03:19 +13:00
break ; // leave the do-while if successful
}
} catch ( \Throwable $th ) {
Console :: warning ( " InfluxDB not ready. Retrying connection ( { $attempts } )... " );
if ( $attempts >= $max ) {
throw new \Exception ( 'InfluxDB database not ready yet' );
2021-09-10 22:23:49 +12:00
}
2022-02-20 23:03:19 +13:00
sleep ( $sleep );
}
} while ( $attempts < $max );
2021-08-30 21:35:32 +12:00
2022-02-20 23:03:19 +13:00
// sync data
foreach ( $globalMetrics as $metric => $options ) { //for each metrics
foreach ( $periods as $period ) { // aggregate data for each period
$start = DateTime :: createFromFormat ( 'U' , \strtotime ( $period [ 'startTime' ])) -> format ( DateTime :: RFC3339 );
if ( ! empty ( $latestTime [ $metric ][ $period [ 'key' ]])) {
$start = DateTime :: createFromFormat ( 'U' , $latestTime [ $metric ][ $period [ 'key' ]]) -> format ( DateTime :: RFC3339 );
}
$end = DateTime :: createFromFormat ( 'U' , \strtotime ( 'now' )) -> format ( DateTime :: RFC3339 );
2021-08-16 18:58:34 +12:00
2022-02-20 23:03:19 +13:00
$table = $options [ 'table' ]; //Which influxdb table to query for this metric
$groupBy = empty ( $options [ 'groupBy' ]) ? '' : ', "' . $options [ 'groupBy' ] . '"' ; //Some sub level metrics may be grouped by other tags like collectionId, bucketId, etc
2021-08-16 18:58:34 +12:00
2022-02-20 23:03:19 +13:00
$filters = $options [ 'filters' ] ? ? []; // Some metrics might have additional filters, like function's status
if ( ! empty ( $filters )) {
$filters = ' AND ' . implode ( ' AND ' , array_map ( fn ( $filter , $value ) => " \" { $filter } \" =' { $value } ' " , array_keys ( $filters ), array_values ( $filters )));
} else {
$filters = '' ;
}
$query = " SELECT sum(value) AS \" value \" FROM \" { $table } \" WHERE \" time \" > ' { $start } ' AND \" time \" < ' { $end } ' AND \" metric_type \" ='counter' { $filters } GROUP BY time( { $period [ 'key' ] } ), \" projectId \" { $groupBy } FILL(null) " ;
try {
$result = $database -> query ( $query );
$points = $result -> getPoints ();
foreach ( $points as $point ) {
$projectId = $point [ 'projectId' ];
2021-08-20 17:55:23 +12:00
2022-02-20 23:03:19 +13:00
if ( ! empty ( $projectId ) && $projectId !== 'console' ) {
2022-02-23 03:16:17 +13:00
$dbForProject -> setNamespace ( '_' . $projectId );
2022-02-20 23:03:19 +13:00
$metricUpdated = $metric ;
if ( ! empty ( $groupBy )) {
$groupedBy = $point [ $options [ 'groupBy' ]] ? ? '' ;
if ( empty ( $groupedBy )) {
continue ;
2021-08-16 18:58:34 +12:00
}
2022-02-20 23:03:19 +13:00
$metricUpdated = str_replace ( $options [ 'groupBy' ], $groupedBy , $metric );
}
$time = \strtotime ( $point [ 'time' ]);
$id = \md5 ( $time . '_' . $period [ 'key' ] . '_' . $metricUpdated ); //Construct unique id for each metric using time, period and metric
$value = ( ! empty ( $point [ 'value' ])) ? $point [ 'value' ] : 0 ;
try {
$document = $dbForProject -> getDocument ( 'stats' , $id );
if ( $document -> isEmpty ()) {
$dbForProject -> createDocument ( 'stats' , new Document ([
'$id' => $id ,
'period' => $period [ 'key' ],
'time' => $time ,
'metric' => $metricUpdated ,
'value' => $value ,
'type' => 0 ,
]));
} else {
$dbForProject -> updateDocument (
'stats' ,
$document -> getId (),
$document -> setAttribute ( 'value' , $value )
);
2021-08-16 18:58:34 +12:00
}
2022-02-20 23:03:19 +13:00
$latestTime [ $metric ][ $period [ 'key' ]] = $time ;
} catch ( \Exception $e ) { // if projects are deleted this might fail
Console :: warning ( " Failed to save data for project { $projectId } and metric { $metricUpdated } : { $e -> getMessage () } " );
2022-02-24 20:40:12 +13:00
Console :: warning ( $e -> getTraceAsString ());
2021-08-16 18:58:34 +12:00
}
}
}
2022-02-20 23:03:19 +13:00
} catch ( \Exception $e ) {
Console :: warning ( " Failed to Query: { $e -> getMessage () } " );
2022-02-24 20:40:12 +13:00
Console :: warning ( $e -> getTraceAsString ());
2021-08-16 18:58:34 +12:00
}
2021-08-11 22:15:00 +12:00
}
}
2021-08-16 20:53:34 +12:00
2022-02-20 22:33:01 +13:00
if ( $iterations % 30 != 0 ) { // Aggregate aggregate number of objects in database only after 15 minutes
$iterations ++ ;
$loopTook = microtime ( true ) - $loopStart ;
$now = date ( 'd-m-Y H:i:s' , time ());
2022-02-20 22:38:52 +13:00
Console :: info ( " [ { $now } ] Aggregation took { $loopTook } seconds " );
2022-02-20 22:33:01 +13:00
return ;
2022-02-20 22:38:52 +13:00
}
2022-02-20 22:33:01 +13:00
2022-02-20 22:38:52 +13:00
/**
* Aggregate MariaDB every 15 minutes
* Some of the queries here might contain full - table scans .
*/
$now = date ( 'd-m-Y H:i:s' , time ());
Console :: info ( " [ { $now } ] Aggregating database counters. " );
2022-05-24 02:54:50 +12:00
2022-02-20 22:38:52 +13:00
$latestProject = null ;
2022-02-20 22:33:01 +13:00
do { // Loop over all the projects
$attempts = 0 ;
$max = 10 ;
$sleep = 1 ;
2021-09-10 22:23:49 +12:00
2022-02-20 22:33:01 +13:00
do { // list projects
try {
$attempts ++ ;
$projects = $dbForConsole -> find ( 'projects' , [], 100 , cursor : $latestProject );
break ; // leave the do-while if successful
} catch ( \Exception $e ) {
Console :: warning ( " Console DB not ready yet. Retrying ( { $attempts } )... " );
if ( $attempts >= $max ) {
throw new \Exception ( 'Failed access console db: ' . $e -> getMessage ());
2021-09-10 22:23:49 +12:00
}
2022-02-20 22:33:01 +13:00
sleep ( $sleep );
2021-08-30 19:19:29 +12:00
}
2022-02-20 22:33:01 +13:00
} while ( $attempts < $max );
2021-08-30 19:19:29 +12:00
2022-02-20 22:33:01 +13:00
if ( empty ( $projects )) {
continue ;
}
2021-10-28 01:12:06 +13:00
2022-02-20 22:33:01 +13:00
$latestProject = $projects [ array_key_last ( $projects )];
foreach ( $projects as $project ) {
$projectId = $project -> getId ();
// Get total storage
2022-02-23 03:16:17 +13:00
$dbForProject -> setNamespace ( '_' . $projectId );
2022-04-21 22:30:32 +12:00
$deploymentsTotal = $dbForProject -> sum ( 'deployments' , 'size' );
2022-02-20 22:33:01 +13:00
$time = ( int ) ( floor ( time () / 1800 ) * 1800 ); // Time rounded to nearest 30 minutes
2022-02-24 20:28:49 +13:00
$id = \md5 ( $time . '_30m_storage.deployments.total' ); //Construct unique id for each metric using time, period and metric
2022-02-20 22:33:01 +13:00
$document = $dbForProject -> getDocument ( 'stats' , $id );
2022-02-24 20:40:12 +13:00
try {
if ( $document -> isEmpty ()) {
$dbForProject -> createDocument ( 'stats' , new Document ([
'$id' => $id ,
'period' => '30m' ,
'time' => $time ,
'metric' => 'storage.deployments.total' ,
2022-04-21 22:30:32 +12:00
'value' => $deploymentsTotal ,
2022-02-24 20:40:12 +13:00
'type' => 1 ,
]));
} else {
$dbForProject -> updateDocument (
'stats' ,
$document -> getId (),
2022-04-21 22:30:32 +12:00
$document -> setAttribute ( 'value' , $deploymentsTotal )
2022-02-24 20:40:12 +13:00
);
}
$time = ( int ) ( floor ( time () / 86400 ) * 86400 ); // Time rounded to nearest day
$id = \md5 ( $time . '_1d_storage.deployments.total' ); //Construct unique id for each metric using time, period and metric
$document = $dbForProject -> getDocument ( 'stats' , $id );
if ( $document -> isEmpty ()) {
$dbForProject -> createDocument ( 'stats' , new Document ([
'$id' => $id ,
'period' => '1d' ,
'time' => $time ,
'metric' => 'storage.deployments.total' ,
2022-04-21 22:30:32 +12:00
'value' => $deploymentsTotal ,
2022-02-24 20:40:12 +13:00
'type' => 1 ,
]));
} else {
$dbForProject -> updateDocument (
'stats' ,
$document -> getId (),
2022-04-21 22:30:32 +12:00
$document -> setAttribute ( 'value' , $deploymentsTotal )
2022-02-24 20:40:12 +13:00
);
}
2022-05-24 02:54:50 +12:00
} catch ( \Exception $e ) {
2022-02-24 20:40:12 +13:00
Console :: warning ( " Failed to save data for project { $projectId } and metric storage.deployments.total: { $e -> getMessage () } " );
Console :: warning ( $e -> getTraceAsString ());
2022-02-20 22:33:01 +13:00
}
2022-02-24 20:40:12 +13:00
2022-02-20 22:33:01 +13:00
$collections = [
'users' => [
'namespace' => '' ,
],
'collections' => [
'metricPrefix' => 'database' ,
'namespace' => '' ,
'subCollections' => [ // Some collections, like collections and later buckets have child collections that need counting
'documents' => [
2022-02-20 23:17:57 +13:00
'collectionPrefix' => 'collection_' ,
2022-02-20 22:33:01 +13:00
'namespace' => '' ,
2021-08-19 20:01:22 +12:00
],
2021-08-30 19:19:29 +12:00
],
2022-02-20 22:33:01 +13:00
],
'buckets' => [
'metricPrefix' => 'storage' ,
'namespace' => '' ,
'subCollections' => [
'files' => [
'namespace' => '' ,
'collectionPrefix' => 'bucket_' ,
2022-02-27 22:57:09 +13:00
'total' => [
2022-02-20 22:33:01 +13:00
'field' => 'sizeOriginal'
]
],
2021-09-10 20:49:16 +12:00
]
2022-02-20 22:33:01 +13:00
]
];
foreach ( $collections as $collection => $options ) {
try {
2022-02-23 03:16:17 +13:00
$dbForProject -> setNamespace ( " _ { $projectId } " );
2022-02-20 22:33:01 +13:00
$count = $dbForProject -> count ( $collection );
$metricPrefix = $options [ 'metricPrefix' ] ? ? '' ;
$metric = empty ( $metricPrefix ) ? " { $collection } .count " : " { $metricPrefix } . { $collection } .count " ;
$time = ( int ) ( floor ( time () / 1800 ) * 1800 ); // Time rounded to nearest 30 minutes
$id = \md5 ( $time . '_30m_' . $metric ); //Construct unique id for each metric using time, period and metric
$document = $dbForProject -> getDocument ( 'stats' , $id );
if ( $document -> isEmpty ()) {
$dbForProject -> createDocument ( 'stats' , new Document ([
'$id' => $id ,
'time' => $time ,
'period' => '30m' ,
'metric' => $metric ,
'value' => $count ,
'type' => 1 ,
]));
} else {
$dbForProject -> updateDocument (
'stats' ,
$document -> getId (),
$document -> setAttribute ( 'value' , $count )
);
}
$time = ( int ) ( floor ( time () / 86400 ) * 86400 ); // Time rounded to nearest day
$id = \md5 ( $time . '_1d_' . $metric ); //Construct unique id for each metric using time, period and metric
$document = $dbForProject -> getDocument ( 'stats' , $id );
if ( $document -> isEmpty ()) {
$dbForProject -> createDocument ( 'stats' , new Document ([
'$id' => $id ,
'time' => $time ,
'period' => '1d' ,
'metric' => $metric ,
'value' => $count ,
'type' => 1 ,
]));
} else {
$dbForProject -> updateDocument (
'stats' ,
$document -> getId (),
$document -> setAttribute ( 'value' , $count )
);
}
$subCollections = $options [ 'subCollections' ] ? ? [];
if ( empty ( $subCollections )) {
continue ;
}
$latestParent = null ;
$subCollectionCounts = []; //total project level count of sub collections
$subCollectionTotals = []; //total project level sum of sub collections
do { // Loop over all the parent collection document for each sub collection
2022-02-08 01:25:15 +13:00
$dbForProject -> setNamespace ( " _ { $projectId } " );
2022-02-20 22:33:01 +13:00
$parents = $dbForProject -> find ( $collection , [], 100 , cursor : $latestParent ); // Get all the parents for the sub collections for example for documents, this will get all the collections
if ( empty ( $parents )) {
continue ;
}
$latestParent = $parents [ array_key_last ( $parents )];
foreach ( $parents as $parent ) {
foreach ( $subCollections as $subCollection => $subOptions ) { // Sub collection counts, like database.collections.collectionId.documents.count
2022-02-23 03:16:17 +13:00
$dbForProject -> setNamespace ( " _ { $projectId } " );
2022-03-27 18:05:47 +13:00
$count = $dbForProject -> count (( $subOptions [ 'collectionPrefix' ] ? ? '' ) . $parent -> getInternalId ());
2022-02-20 22:33:01 +13:00
$subCollectionCounts [ $subCollection ] = ( $subCollectionCounts [ $subCollection ] ? ? 0 ) + $count ; // Project level counts for sub collections like database.documents.count
2022-02-23 03:16:17 +13:00
$dbForProject -> setNamespace ( " _ { $projectId } " );
2022-02-20 22:33:01 +13:00
2022-03-27 18:05:47 +13:00
$metric = empty ( $metricPrefix ) ? " { $collection } . { $parent -> getId () } . { $subCollection } .count " : " { $metricPrefix } . { $collection } . { $parent -> getInternalId () } . { $subCollection } .count " ;
2022-02-20 22:33:01 +13:00
$time = ( int ) ( floor ( time () / 1800 ) * 1800 ); // Time rounded to nearest 30 minutes
$id = \md5 ( $time . '_30m_' . $metric ); //Construct unique id for each metric using time, period and metric
$document = $dbForProject -> getDocument ( 'stats' , $id );
if ( $document -> isEmpty ()) {
$dbForProject -> createDocument ( 'stats' , new Document ([
'$id' => $id ,
'time' => $time ,
'period' => '30m' ,
'metric' => $metric ,
'value' => $count ,
'type' => 1 ,
]));
} else {
$dbForProject -> updateDocument (
'stats' ,
$document -> getId (),
$document -> setAttribute ( 'value' , $count )
);
}
$time = ( int ) ( floor ( time () / 86400 ) * 86400 ); // Time rounded to nearest day
$id = \md5 ( $time . '_1d_' . $metric ); //Construct unique id for each metric using time, period and metric
$document = $dbForProject -> getDocument ( 'stats' , $id );
if ( $document -> isEmpty ()) {
$dbForProject -> createDocument ( 'stats' , new Document ([
'$id' => $id ,
'time' => $time ,
'period' => '1d' ,
'metric' => $metric ,
'value' => $count ,
'type' => 1 ,
]));
} else {
$dbForProject -> updateDocument (
'stats' ,
$document -> getId (),
$document -> setAttribute ( 'value' , $count )
);
}
// check if sum calculation is required
2022-02-27 22:57:09 +13:00
$total = $subOptions [ 'total' ] ? ? [];
2022-05-24 02:54:50 +12:00
if ( empty ( $total )) {
2022-02-20 22:33:01 +13:00
continue ;
}
2022-02-23 03:16:17 +13:00
$dbForProject -> setNamespace ( " _ { $projectId } " );
2022-03-27 18:05:47 +13:00
$total = ( int ) $dbForProject -> sum (( $subOptions [ 'collectionPrefix' ] ? ? '' ) . $parent -> getInternalId (), $total [ 'field' ]);
2022-02-20 22:33:01 +13:00
$subCollectionTotals [ $subCollection ] = ( $ssubCollectionTotals [ $subCollection ] ? ? 0 ) + $total ; // Project level sum for sub collections like storage.total
2022-02-23 03:16:17 +13:00
$dbForProject -> setNamespace ( " _ { $projectId } " );
2022-02-20 22:33:01 +13:00
2022-03-27 18:05:47 +13:00
$metric = empty ( $metricPrefix ) ? " { $collection } . { $parent -> getId () } . { $subCollection } .total " : " { $metricPrefix } . { $collection } . { $parent -> getInternalId () } . { $subCollection } .total " ;
2022-02-20 22:33:01 +13:00
$time = ( int ) ( floor ( time () / 1800 ) * 1800 ); // Time rounded to nearest 30 minutes
$id = \md5 ( $time . '_30m_' . $metric ); //Construct unique id for each metric using time, period and metric
$document = $dbForProject -> getDocument ( 'stats' , $id );
if ( $document -> isEmpty ()) {
$dbForProject -> createDocument ( 'stats' , new Document ([
'$id' => $id ,
'time' => $time ,
'period' => '30m' ,
'metric' => $metric ,
'value' => $total ,
'type' => 1 ,
]));
} else {
2022-05-24 02:54:50 +12:00
$dbForProject -> updateDocument (
'stats' ,
$document -> getId (),
$document -> setAttribute ( 'value' , $total )
);
2022-02-20 22:33:01 +13:00
}
$time = ( int ) ( floor ( time () / 86400 ) * 86400 ); // Time rounded to nearest day
$id = \md5 ( $time . '_1d_' . $metric ); //Construct unique id for each metric using time, period and metric
$document = $dbForProject -> getDocument ( 'stats' , $id );
if ( $document -> isEmpty ()) {
$dbForProject -> createDocument ( 'stats' , new Document ([
'$id' => $id ,
'time' => $time ,
'period' => '1d' ,
'metric' => $metric ,
'value' => $total ,
'type' => 1 ,
]));
} else {
2022-05-24 02:54:50 +12:00
$dbForProject -> updateDocument (
'stats' ,
$document -> getId (),
$document -> setAttribute ( 'value' , $total )
);
2022-02-20 22:33:01 +13:00
}
}
}
} while ( ! empty ( $parents ));
/**
* Inserting project level counts for sub collections like database . documents . count
*/
foreach ( $subCollectionCounts as $subCollection => $count ) {
2022-02-23 03:16:17 +13:00
$dbForProject -> setNamespace ( " _ { $projectId } " );
2022-02-20 22:33:01 +13:00
$metric = empty ( $metricPrefix ) ? " { $subCollection } .count " : " { $metricPrefix } . { $subCollection } .count " ;
2021-08-30 21:35:32 +12:00
$time = ( int ) ( floor ( time () / 1800 ) * 1800 ); // Time rounded to nearest 30 minutes
$id = \md5 ( $time . '_30m_' . $metric ); //Construct unique id for each metric using time, period and metric
$document = $dbForProject -> getDocument ( 'stats' , $id );
if ( $document -> isEmpty ()) {
$dbForProject -> createDocument ( 'stats' , new Document ([
'$id' => $id ,
'time' => $time ,
'period' => '30m' ,
'metric' => $metric ,
'value' => $count ,
'type' => 1 ,
]));
} else {
2021-10-28 01:12:06 +13:00
$dbForProject -> updateDocument (
'stats' ,
$document -> getId (),
$document -> setAttribute ( 'value' , $count )
);
2021-08-30 21:35:32 +12:00
}
$time = ( int ) ( floor ( time () / 86400 ) * 86400 ); // Time rounded to nearest day
$id = \md5 ( $time . '_1d_' . $metric ); //Construct unique id for each metric using time, period and metric
$document = $dbForProject -> getDocument ( 'stats' , $id );
if ( $document -> isEmpty ()) {
$dbForProject -> createDocument ( 'stats' , new Document ([
'$id' => $id ,
'time' => $time ,
'period' => '1d' ,
'metric' => $metric ,
'value' => $count ,
'type' => 1 ,
]));
} else {
2021-10-28 01:12:06 +13:00
$dbForProject -> updateDocument (
'stats' ,
$document -> getId (),
$document -> setAttribute ( 'value' , $count )
);
2021-08-30 21:35:32 +12:00
}
2022-02-20 22:33:01 +13:00
}
2021-08-30 19:19:29 +12:00
2022-02-20 22:33:01 +13:00
/**
2022-04-21 22:30:32 +12:00
* Inserting project level sums for sub collections like storage . files . total
2022-02-20 22:33:01 +13:00
*/
foreach ( $subCollectionTotals as $subCollection => $count ) {
2022-02-23 03:16:17 +13:00
$dbForProject -> setNamespace ( " _ { $projectId } " );
2021-08-30 19:19:29 +12:00
2022-02-20 22:33:01 +13:00
$metric = empty ( $metricPrefix ) ? " { $subCollection } .total " : " { $metricPrefix } . { $subCollection } .total " ;
2021-11-09 18:50:03 +13:00
2022-02-20 22:33:01 +13:00
$time = ( int ) ( floor ( time () / 1800 ) * 1800 ); // Time rounded to nearest 30 minutes
$id = \md5 ( $time . '_30m_' . $metric ); //Construct unique id for each metric using time, period and metric
$document = $dbForProject -> getDocument ( 'stats' , $id );
if ( $document -> isEmpty ()) {
$dbForProject -> createDocument ( 'stats' , new Document ([
'$id' => $id ,
'time' => $time ,
'period' => '30m' ,
'metric' => $metric ,
'value' => $count ,
'type' => 1 ,
]));
} else {
2022-05-24 02:54:50 +12:00
$dbForProject -> updateDocument (
'stats' ,
$document -> getId (),
$document -> setAttribute ( 'value' , $count )
);
2021-11-09 18:50:03 +13:00
}
2022-02-20 22:33:01 +13:00
$time = ( int ) ( floor ( time () / 86400 ) * 86400 ); // Time rounded to nearest day
$id = \md5 ( $time . '_1d_' . $metric ); //Construct unique id for each metric using time, period and metric
$document = $dbForProject -> getDocument ( 'stats' , $id );
if ( $document -> isEmpty ()) {
$dbForProject -> createDocument ( 'stats' , new Document ([
'$id' => $id ,
'time' => $time ,
'period' => '1d' ,
'metric' => $metric ,
'value' => $count ,
'type' => 1 ,
]));
} else {
2022-05-24 02:54:50 +12:00
$dbForProject -> updateDocument (
'stats' ,
$document -> getId (),
$document -> setAttribute ( 'value' , $count )
);
2021-08-19 20:01:22 +12:00
}
2022-04-21 22:30:32 +12:00
// aggregate storage.total = storage.files.total + storage.deployments.total
2022-05-24 02:54:50 +12:00
if ( $metricPrefix === 'storage' && $subCollection === 'files' ) {
2022-04-21 22:30:32 +12:00
$metric = 'storage.total' ;
$time = ( int ) ( floor ( time () / 1800 ) * 1800 ); // Time rounded to nearest 30 minutes
$id = \md5 ( $time . '_30m_' . $metric ); //Construct unique id for each metric using time, period and metric
$document = $dbForProject -> getDocument ( 'stats' , $id );
if ( $document -> isEmpty ()) {
$dbForProject -> createDocument ( 'stats' , new Document ([
'$id' => $id ,
'time' => $time ,
'period' => '30m' ,
'metric' => $metric ,
'value' => $count + $deploymentsTotal ,
'type' => 1 ,
]));
} else {
2022-05-24 02:54:50 +12:00
$dbForProject -> updateDocument (
'stats' ,
$document -> getId (),
$document -> setAttribute ( 'value' , $count + $deploymentsTotal )
);
2022-04-21 22:30:32 +12:00
}
$time = ( int ) ( floor ( time () / 86400 ) * 86400 ); // Time rounded to nearest day
$id = \md5 ( $time . '_1d_' . $metric ); //Construct unique id for each metric using time, period and metric
$document = $dbForProject -> getDocument ( 'stats' , $id );
if ( $document -> isEmpty ()) {
$dbForProject -> createDocument ( 'stats' , new Document ([
'$id' => $id ,
'time' => $time ,
'period' => '1d' ,
'metric' => $metric ,
'value' => $count + $deploymentsTotal ,
'type' => 1 ,
]));
} else {
2022-05-24 02:54:50 +12:00
$dbForProject -> updateDocument (
'stats' ,
$document -> getId (),
$document -> setAttribute ( 'value' , $count + $deploymentsTotal )
);
2022-04-21 22:30:32 +12:00
}
}
2021-08-17 18:03:27 +12:00
}
2022-02-20 22:33:01 +13:00
} catch ( \Exception $e ) {
Console :: warning ( " Failed to save database counters data for project { $collection } : { $e -> getMessage () } " );
2022-02-24 20:40:12 +13:00
Console :: warning ( $e -> getTraceAsString ());
2021-08-17 17:45:07 +12:00
}
}
2022-02-20 22:33:01 +13:00
}
} while ( ! empty ( $projects ));
2021-08-30 19:19:29 +12:00
2021-08-17 18:03:27 +12:00
$iterations ++ ;
2021-08-17 17:45:07 +12:00
$loopTook = microtime ( true ) - $loopStart ;
$now = date ( 'd-m-Y H:i:s' , time ());
2021-08-30 21:35:32 +12:00
2021-08-17 18:03:27 +12:00
Console :: info ( " [ { $now } ] Aggregation took { $loopTook } seconds " );
}, $interval );
2022-05-24 02:54:50 +12:00
});