2022-06-13 22:56:24 +12:00
< ? php
namespace Appwrite\Stats ;
use Utopia\Database\Database ;
use Utopia\Database\Document ;
use InfluxDB\Database as InfluxDatabase ;
use DateTime ;
2022-06-14 03:49:27 +12:00
class Usage
{
2022-06-13 22:56:24 +12:00
protected InfluxDatabase $influxDB ;
protected Database $database ;
2022-06-13 23:11:26 +12:00
protected $errorHandler ;
private array $latestTime = [];
2022-06-13 22:56:24 +12:00
// all the mertics that we are collecting
protected array $metrics = [
'requests' => [
'table' => 'appwrite_usage_requests_all' ,
],
'network' => [
'table' => 'appwrite_usage_network_all' ,
],
'executions' => [
'table' => 'appwrite_usage_executions_all' ,
],
2022-06-22 22:51:49 +12:00
'databases.create' => [
'table' => 'appwrite_usage_databases_create' ,
2022-06-13 22:56:24 +12:00
],
2022-06-22 22:51:49 +12:00
'databases.read' => [
'table' => 'appwrite_usage_databases_read' ,
2022-06-13 22:56:24 +12:00
],
2022-06-22 22:51:49 +12:00
'databases.update' => [
'table' => 'appwrite_usage_databases_update' ,
2022-06-13 22:56:24 +12:00
],
2022-06-22 22:51:49 +12:00
'databases.delete' => [
'table' => 'appwrite_usage_databases_delete' ,
2022-06-13 22:56:24 +12:00
],
2022-06-22 22:51:49 +12:00
'databases.collections.create' => [
'table' => 'appwrite_usage_databases_collections_create' ,
2022-06-13 22:56:24 +12:00
],
2022-06-22 22:51:49 +12:00
'databases.collections.read' => [
'table' => 'appwrite_usage_databases_collections_read' ,
2022-06-13 22:56:24 +12:00
],
2022-06-22 22:51:49 +12:00
'databases.collections.update' => [
'table' => 'appwrite_usage_databases_collections_update' ,
2022-06-13 22:56:24 +12:00
],
2022-06-22 22:51:49 +12:00
'databases.collections.delete' => [
'table' => 'appwrite_usage_databases_collections_delete' ,
2022-06-13 22:56:24 +12:00
],
2022-06-22 22:51:49 +12:00
'databases.documents.create' => [
'table' => 'appwrite_usage_databases_documents_create' ,
2022-06-13 22:56:24 +12:00
],
2022-06-22 22:51:49 +12:00
'databases.documents.read' => [
'table' => 'appwrite_usage_databases_documents_read' ,
2022-06-13 22:56:24 +12:00
],
2022-06-22 22:51:49 +12:00
'databases.documents.update' => [
'table' => 'appwrite_usage_databases_documents_update' ,
2022-06-13 22:56:24 +12:00
],
2022-06-22 22:51:49 +12:00
'databases.documents.delete' => [
'table' => 'appwrite_usage_databases_documents_delete' ,
],
'databases.databaseId.collections.create' => [
'table' => 'appwrite_usage_databases_collections_create' ,
'groupBy' => [ 'databaseId' ],
],
'databases.databaseId.collections.read' => [
'table' => 'appwrite_usage_databases_collections_read' ,
'groupBy' => [ 'databaseId' ],
],
'databases.databaseId.collections.update' => [
'table' => 'appwrite_usage_databases_collections_update' ,
'groupBy' => [ 'databaseId' ],
],
'databases.databaseId.collections.delete' => [
'table' => 'appwrite_usage_databases_collections_delete' ,
'groupBy' => [ 'databaseId' ],
],
'databases.databaseId.documents.create' => [
'table' => 'appwrite_usage_databases_documents_create' ,
'groupBy' => [ 'databaseId' ],
],
'databases.databaseId.documents.read' => [
'table' => 'appwrite_usage_databases_documents_read' ,
'groupBy' => [ 'databaseId' ],
],
'database.databaseId.documents.update' => [
'table' => 'appwrite_usage_databases_documents_update' ,
'groupBy' => [ 'databaseId' ],
],
'databases.databaseId.documents.delete' => [
'table' => 'appwrite_usage_databases_documents_delete' ,
'groupBy' => [ 'databaseId' ],
],
'databases.databaseId.collections.collectionId.documents.create' => [
'table' => 'appwrite_usage_databases_documents_create' ,
'groupBy' => [ 'collectionId' ],
],
'databases.databaseId.collections.collectionId.documents.read' => [
'table' => 'appwrite_usage_databases_documents_read' ,
'groupBy' => [ 'databaseId' , 'collectionId' ],
],
'databases.databaseId.collections.collectionId.documents.update' => [
'table' => 'appwrite_usage_databases_documents_update' ,
'groupBy' => [ 'databaseId' , 'collectionId' ],
],
'databases.databaseId.collections.collectionId.documents.delete' => [
'table' => 'appwrite_usage_databases_documents_delete' ,
'groupBy' => [ 'databaseId' , 'collectionId' ],
2022-06-13 22:56:24 +12:00
],
'storage.buckets.create' => [
'table' => 'appwrite_usage_storage_buckets_create' ,
],
'storage.buckets.read' => [
'table' => 'appwrite_usage_storage_buckets_read' ,
],
'storage.buckets.update' => [
'table' => 'appwrite_usage_storage_buckets_update' ,
],
'storage.buckets.delete' => [
'table' => 'appwrite_usage_storage_buckets_delete' ,
],
'storage.files.create' => [
'table' => 'appwrite_usage_storage_files_create' ,
],
'storage.files.read' => [
'table' => 'appwrite_usage_storage_files_read' ,
],
'storage.files.update' => [
'table' => 'appwrite_usage_storage_files_update' ,
],
'storage.files.delete' => [
'table' => 'appwrite_usage_storage_files_delete' ,
],
'storage.buckets.bucketId.files.create' => [
'table' => 'appwrite_usage_storage_files_create' ,
2022-06-22 22:51:49 +12:00
'groupBy' => [ 'bucketId' ],
2022-06-13 22:56:24 +12:00
],
'storage.buckets.bucketId.files.read' => [
'table' => 'appwrite_usage_storage_files_read' ,
2022-06-22 22:51:49 +12:00
'groupBy' => [ 'bucketId' ],
2022-06-13 22:56:24 +12:00
],
'storage.buckets.bucketId.files.update' => [
'table' => 'appwrite_usage_storage_files_update' ,
2022-06-22 22:51:49 +12:00
'groupBy' => [ 'bucketId' ],
2022-06-13 22:56:24 +12:00
],
'storage.buckets.bucketId.files.delete' => [
'table' => 'appwrite_usage_storage_files_delete' ,
2022-06-22 22:51:49 +12:00
'groupBy' => [ 'bucketId' ],
2022-06-13 22:56:24 +12:00
],
'users.create' => [
'table' => 'appwrite_usage_users_create' ,
],
'users.read' => [
'table' => 'appwrite_usage_users_read' ,
],
'users.update' => [
'table' => 'appwrite_usage_users_update' ,
],
'users.delete' => [
'table' => 'appwrite_usage_users_delete' ,
],
'users.sessions.create' => [
'table' => 'appwrite_usage_users_sessions_create' ,
],
'users.sessions.provider.create' => [
'table' => 'appwrite_usage_users_sessions_create' ,
2022-06-22 22:51:49 +12:00
'groupBy' => [ 'provider' ],
2022-06-13 22:56:24 +12:00
],
'users.sessions.delete' => [
'table' => 'appwrite_usage_users_sessions_delete' ,
],
'functions.functionId.executions' => [
'table' => 'appwrite_usage_executions_all' ,
2022-06-22 22:51:49 +12:00
'groupBy' => [ 'functionId' ],
2022-06-13 22:56:24 +12:00
],
'functions.functionId.compute' => [
'table' => 'appwrite_usage_executions_time' ,
2022-06-22 22:51:49 +12:00
'groupBy' => [ 'functionId' ],
2022-06-13 22:56:24 +12:00
],
'functions.functionId.failures' => [
'table' => 'appwrite_usage_executions_all' ,
2022-06-22 22:51:49 +12:00
'groupBy' => [ 'functionId' ],
2022-06-13 22:56:24 +12:00
'filters' => [
'functionStatus' => 'failed' ,
],
],
];
protected array $periods = [
[
'key' => '30m' ,
2022-06-13 23:11:26 +12:00
'multiplier' => 1800 ,
2022-06-13 22:56:24 +12:00
'startTime' => '-24 hours' ,
],
[
'key' => '1d' ,
2022-06-13 23:11:26 +12:00
'multiplier' => 86400 ,
2022-06-13 22:56:24 +12:00
'startTime' => '-90 days' ,
],
];
2022-06-13 23:11:26 +12:00
public function __construct ( Database $database , InfluxDatabase $influxDB , callable $errorHandler = null )
2022-06-13 22:56:24 +12:00
{
$this -> database = $database ;
$this -> influxDB = $influxDB ;
2022-06-13 23:11:26 +12:00
$this -> errorHandler = $errorHandler ;
2022-06-13 22:56:24 +12:00
}
/**
* Create or Update Mertic
* Create or update each metric in the stats collection for the given project
2022-06-14 03:49:27 +12:00
*
2022-06-13 22:56:24 +12:00
* @ param string $projectId
* @ param int $time
* @ param string $period
* @ param string $metric
* @ param int $value
* @ param int $type
2022-06-14 03:49:27 +12:00
*
2022-06-13 22:56:24 +12:00
* @ return void
*/
2022-06-13 23:11:26 +12:00
private function createOrUpdateMetric ( string $projectId , int $time , string $period , string $metric , int $value , int $type ) : void
2022-06-13 22:56:24 +12:00
{
$id = \md5 ( " { $time } _ { $period } _ { $metric } " );
2022-06-20 23:21:36 +12:00
$this -> database -> setNamespace ( '_console' );
$project = $this -> database -> getDocument ( 'projects' , $projectId );
$this -> database -> setNamespace ( '_' . $project -> getInternalId ());
2022-06-13 22:56:24 +12:00
try {
$document = $this -> database -> getDocument ( 'stats' , $id );
if ( $document -> isEmpty ()) {
$this -> database -> createDocument ( 'stats' , new Document ([
'$id' => $id ,
2022-06-14 12:40:33 +12:00
'period' => $period ,
2022-06-13 22:56:24 +12:00
'time' => $time ,
'metric' => $metric ,
'value' => $value ,
'type' => $type ,
]));
} else {
$this -> database -> updateDocument (
'stats' ,
$document -> getId (),
$document -> setAttribute ( 'value' , $value )
);
}
2022-06-14 12:40:33 +12:00
$this -> latestTime [ $metric ][ $period ] = $time ;
2022-06-13 22:56:24 +12:00
} catch ( \Exception $e ) { // if projects are deleted this might fail
2022-06-14 03:49:27 +12:00
if ( is_callable ( $this -> errorHandler )) {
2022-06-15 11:40:56 +12:00
call_user_func ( $this -> errorHandler , $e , " sync_project_ { $projectId } _metric_ { $metric } " );
2022-06-13 23:11:26 +12:00
} else {
throw $e ;
}
2022-06-13 22:56:24 +12:00
}
}
/**
* Sync From InfluxDB
* Sync stats from influxDB to stats collection in the Appwrite database
2022-06-14 03:49:27 +12:00
*
2022-06-13 22:56:24 +12:00
* @ param string $metric
* @ param array $options
* @ param array $period
2022-06-14 03:49:27 +12:00
*
2022-06-13 22:56:24 +12:00
* @ return void
*/
2022-06-13 23:11:26 +12:00
private function syncFromInfluxDB ( string $metric , array $options , array $period ) : void
2022-06-13 22:56:24 +12:00
{
$start = DateTime :: createFromFormat ( 'U' , \strtotime ( $period [ 'startTime' ])) -> format ( DateTime :: RFC3339 );
2022-06-13 23:11:26 +12:00
if ( ! empty ( $this -> latestTime [ $metric ][ $period [ 'key' ]])) {
$start = DateTime :: createFromFormat ( 'U' , $this -> latestTime [ $metric ][ $period [ 'key' ]]) -> format ( DateTime :: RFC3339 );
2022-06-13 22:56:24 +12:00
}
$end = DateTime :: createFromFormat ( 'U' , \strtotime ( 'now' )) -> format ( DateTime :: RFC3339 );
$table = $options [ 'table' ]; //Which influxdb table to query for this metric
2022-06-22 22:51:49 +12:00
$groupBy = empty ( $options [ 'groupBy' ]) ? '' : ', ' . implode ( ', ' , array_map ( fn ( $groupBy ) => '"' . $groupBy . '" ' , $options [ 'groupBy' ])); //Some sub level metrics may be grouped by other tags like collectionId, bucketId, etc
2022-06-13 22:56:24 +12:00
$filters = $options [ 'filters' ] ? ? []; // Some metrics might have additional filters, like function's status
if ( ! empty ( $filters )) {
$filters = ' AND ' . implode ( ' AND ' , array_map ( fn ( $filter , $value ) => " \" { $filter } \" =' { $value } ' " , array_keys ( $filters ), array_values ( $filters )));
} else {
$filters = '' ;
}
2022-06-15 11:23:23 +12:00
$query = " SELECT sum(value) AS \" value \" " ;
$query .= " FROM \" { $table } \" " ;
$query .= " WHERE \" time \" > ' { $start } ' " ;
$query .= " AND \" time \" < ' { $end } ' " ;
$query .= " AND \" metric_type \" ='counter' { $filters } " ;
$query .= " GROUP BY time( { $period [ 'key' ] } ), \" projectId \" { $groupBy } " ;
$query .= " FILL(null) " ;
2022-06-22 22:51:49 +12:00
2022-06-13 22:56:24 +12:00
$result = $this -> influxDB -> query ( $query );
$points = $result -> getPoints ();
foreach ( $points as $point ) {
$projectId = $point [ 'projectId' ];
if ( ! empty ( $projectId ) && $projectId !== 'console' ) {
$metricUpdated = $metric ;
if ( ! empty ( $groupBy )) {
2022-06-22 22:51:49 +12:00
foreach ( $options [ 'groupBy' ] as $groupBy ) {
$groupedBy = $point [ $groupBy ] ? ? '' ;
if ( empty ( $groupedBy )) {
continue ;
}
$metricUpdated = str_replace ( $groupBy , $groupedBy , $metric );
2022-06-13 22:56:24 +12:00
}
}
$time = \strtotime ( $point [ 'time' ]);
$value = ( ! empty ( $point [ 'value' ])) ? $point [ 'value' ] : 0 ;
$this -> createOrUpdateMetric (
$projectId ,
$time ,
$period [ 'key' ],
$metricUpdated ,
$value ,
0
);
}
}
}
2022-06-13 23:11:26 +12:00
/**
* Collect Stats
* Collect all the stats from Influd DB to Database
2022-06-14 03:49:27 +12:00
*
2022-06-13 23:11:26 +12:00
* @ return void
*/
public function collect () : void
{
foreach ( $this -> metrics as $metric => $options ) { //for each metrics
foreach ( $this -> periods as $period ) { // aggregate data for each period
try {
$this -> syncFromInfluxDB ( $metric , $options , $period );
} catch ( \Exception $e ) {
2022-06-14 03:49:27 +12:00
if ( is_callable ( $this -> errorHandler )) {
2022-06-15 11:40:56 +12:00
call_user_func ( $this -> errorHandler , $e );
2022-06-13 23:11:26 +12:00
} else {
throw $e ;
}
}
}
}
}
2022-06-14 03:49:27 +12:00
}