From 4de148f783307f073d278ae983624663975cfb7b Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Wed, 21 Sep 2022 09:36:42 +0000 Subject: [PATCH 01/69] new stats to count resources --- src/Appwrite/Usage/Stats.php | 41 ++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/src/Appwrite/Usage/Stats.php b/src/Appwrite/Usage/Stats.php index 00298238f..20ab58b68 100644 --- a/src/Appwrite/Usage/Stats.php +++ b/src/Appwrite/Usage/Stats.php @@ -109,6 +109,12 @@ class Stats if ($value >= 1) { $this->statsd->increment($metric . $tags); } + if($metric == $usersMetrics[0]) { + $this->statsd->increment('users.{scope}.count.total' . $tags); + } + if($metric == $usersMetrics[3]) { + $this->statsd->decrement('users.{scope}.count.total' . $tags); + } } $dbMetrics = [ @@ -131,6 +137,27 @@ class Stats if ($value >= 1) { $dbTags = $tags . ",collectionId=" . ($this->params['collectionId'] ?? '') . ",databaseId=" . ($this->params['databaseId'] ?? ''); $this->statsd->increment($metric . $dbTags); + + if($metric == $dbMetrics[0]) { + $this->statsd->increment('databases.{scope}.count.total' . $dbTags); + } + if($metric == $dbMetrics[3]) { + $this->statsd->decrement('databases.{scope}.count.total' . $dbTags); + } + + if($metric == $dbMetrics[4]) { + $this->statsd->increment('collections.{scope}.count.total' . $dbTags); + } + if($metric == $dbMetrics[7]) { + $this->statsd->decrement('collections.{scope}.count.total' . $dbTags); + } + + if($metric == $dbMetrics[8]) { + $this->statsd->increment('documents.{scope}.count.total' . $dbTags); + } + if($metric == $dbMetrics[11]) { + $this->statsd->decrement('documents.{scope}.count.total' . $dbTags); + } } } @@ -150,6 +177,20 @@ class Stats if ($value >= 1) { $storageTags = $tags . ",bucketId=" . ($this->params['bucketId'] ?? ''); $this->statsd->increment($metric . $storageTags); + + if($metric == $storageMertics[0]) { + $this->statsd->increment('buckets.{scope}.count.total' . $storageTags); + } + if($metric == $storageMertics[3]) { + $this->statsd->decrement('buckets.{scope}.count.total' . $storageTags); + } + + if($metric == $storageMertics[4]) { + $this->statsd->increment('files.{scope}.count.total' . $storageTags); + } + if($metric == $storageMertics[7]) { + $this->statsd->decrement('files.{scope}.count.total' . $storageTags); + } } } From 2956d5f3ebde600ee744e320ce87951b0ce2892e Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 22 Sep 2022 07:34:03 +0000 Subject: [PATCH 02/69] collect console stats --- app/controllers/shared/api.php | 1 - 1 file changed, 1 deletion(-) diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index e07f40514..34ab14858 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -404,7 +404,6 @@ App::shutdown() if ( App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled' && $project->getId() - && $mode !== APP_MODE_ADMIN // TODO: add check to make sure user is admin && !empty($route->getLabel('sdk.namespace', null)) ) { // Don't calculate console usage on admin mode $metric = $route->getLabel('usage.metric', ''); From 80502ff937d021b02dbefe818e1160b79efe9f8a Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 22 Sep 2022 07:34:11 +0000 Subject: [PATCH 03/69] collect counter data --- src/Appwrite/Usage/Calculators/TimeSeries.php | 157 +++++++++++++++++- 1 file changed, 156 insertions(+), 1 deletion(-) diff --git a/src/Appwrite/Usage/Calculators/TimeSeries.php b/src/Appwrite/Usage/Calculators/TimeSeries.php index 01c866120..54aff2d9f 100644 --- a/src/Appwrite/Usage/Calculators/TimeSeries.php +++ b/src/Appwrite/Usage/Calculators/TimeSeries.php @@ -273,8 +273,36 @@ class TimeSeries extends Calculator ], ]; + private array $counterMetrics = [ + 'users.$all.count.total' => [ + 'table' => 'appwrite_usage_users_{scope}_count_total', + ], + 'databases.$all.count.total' => [ + 'table' => 'appwrite_usage_databases_{scope}_count_total', + ], + 'collections.$all.count.total' => [ + 'table' => 'appwrite_usage_collections_{scope}_count_total', + ], + 'documents.$all.count.total' => [ + 'table' => 'appwrite_usage_documents_{scope}_count_total', + 'groupBy' => ['databaseId'] + ], + 'collections.databaseId.count.total' => [ + 'table' => 'appwrite_usage_collections_{scope}_count_total', + 'groupBy' => ['databaseId'] + ], + 'documents.databaseId.count.total' => [ + 'table' => 'appwrite_usage_documents_{scope}_count_total', + 'groupBy' => ['databaseId'] + ], + 'documents.databaseId/collectionId.count.total' => [ + 'table' => 'appwrite_usage_documents_{scope}_count_total', + 'groupBy' => ['databaseId', 'collectionId'] + ], + ]; + protected array $period = [ - 'key' => '30m', + 'key' => '1h', 'startTime' => '-24 hours', ]; @@ -331,6 +359,121 @@ class TimeSeries extends Calculator } } } + /** + * Create or Update Count Mertic + * Create or update each metric in the stats collection for the given project + * + * @param string $projectId + * @param int $time + * @param string $period + * @param string $metric + * @param int $value + * @param int $type + * + * @return void + */ + private function createOrUpdateCountMetric(string $projectId, string $time, string $period, string $metric, int $value): void + { + $id = \md5("{$time}_{$period}_{$metric}"); + $this->database->setNamespace('_console'); + $project = $this->database->getDocument('projects', $projectId); + $this->database->setNamespace('_' . $project->getInternalId()); + + try { + $document = $this->database->getDocument('stats', $id); + if ($document->isEmpty()) { + $this->database->createDocument('stats', new Document([ + '$id' => $id, + 'period' => $period, + 'time' => $time, + 'metric' => $metric, + 'value' => $value, + 'type' => 1, + ])); + } else { + $current = $document->getAttribute('value', 0); + $value = $current + $value; + $this->database->updateDocument( + 'stats', + $document->getId(), + $document->setAttribute('value', $value) + ); + } + } catch (\Exception $e) { // if projects are deleted this might fail + if (is_callable($this->errorHandler)) { + call_user_func($this->errorHandler, $e, "sync_project_{$projectId}_metric_{$metric}"); + } else { + throw $e; + } + } + } + + private function syncCounters(string $metric, array $options, array $period) { + $start = null; // need to calculate last updated date + if (!empty($this->latestTime[$metric][$period['key']])) { + $start = $this->latestTime[$metric][$period['key']]; + } + $end = (new DateTime())->format(DateTime::RFC3339); + + $table = $options['table']; + $groupBy = empty($options['groupBy']) ? '' : ', ' . implode(', ', array_map(fn($groupBy) => '"' . $groupBy . '" ', $options['groupBy'])); //Some sub level metrics may be grouped by other tags like collectionId, bucketId, etc + + $filters = $options['filters'] ?? []; // Some metrics might have additional filters, like function's status + if (!empty($filters)) { + $filters = ' AND ' . implode(' AND ', array_map(fn ($filter, $value) => "\"{$filter}\"='{$value}'", array_keys($filters), array_values($filters))); + } else { + $filters = ''; + } + + $query = "SELECT sum(value) AS \"value\" "; + $query .= "FROM \"{$table}\" "; + $query .= "WHERE "; + $query .= is_null($start) ? '' : "\"time\" > '{$start}' "; + $query .= is_null($start) ? '' : "AND "; + $query .= "\"time\" < '{$end}' "; + $query .= "AND \"metric_type\"='counter' {$filters} "; + $query .= "GROUP BY time({$period['key']}), \"projectId\" {$groupBy} "; + $query .= "FILL(null)"; + + try { + $result = $this->influxDB->query($query); + $points = $result->getPoints(); + foreach ($points as $point) { + $projectId = $point['projectId']; + + if (!empty($projectId) && $projectId !== 'console') { + $metricUpdated = $metric; + if (!empty($groupBy)) { + foreach ($options['groupBy'] as $groupBy) { + $groupedBy = $point[$groupBy] ?? ''; + if (empty($groupedBy)) { + continue; + } + $metricUpdated = str_replace($groupBy, $groupedBy, $metricUpdated); + } + } + + $value = (!empty($point['value'])) ? $point['value'] : 0; + + $this->createOrUpdateCountMetric( + $projectId, + $point['time'], + $period['key'], + $metricUpdated, + $value, + 0 + ); + $this->latestTime[$metric][$period['key']] = $point['time']; + } + } + } catch (\Exception $e) { // if projects are deleted this might fail + if (is_callable($this->errorHandler)) { + call_user_func($this->errorHandler, $e, "sync_metric_{$metric}_influxdb"); + } else { + throw $e; + } + } + } /** * Sync From InfluxDB @@ -416,6 +559,18 @@ class TimeSeries extends Calculator */ public function collect(): void { + foreach($this->counterMetrics as $metric => $options) { + try { + $this->syncCounters($metric, $options, $this->period); + } catch (\Exception $e) { + if (is_callable($this->errorHandler)) { + call_user_func($this->errorHandler, $e); + } else { + throw $e; + } + } + } + foreach ($this->metrics as $metric => $options) { //for each metrics try { $this->syncFromInfluxDB($metric, $options, $this->period); From aee87e3d01ca313766797713e526b8b83700e4bf Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 22 Sep 2022 12:19:43 +0000 Subject: [PATCH 04/69] fix stats users --- src/Appwrite/Usage/Stats.php | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Appwrite/Usage/Stats.php b/src/Appwrite/Usage/Stats.php index 20ab58b68..e4d5640ea 100644 --- a/src/Appwrite/Usage/Stats.php +++ b/src/Appwrite/Usage/Stats.php @@ -108,12 +108,12 @@ class Stats $value = $this->params[$metric] ?? 0; if ($value >= 1) { $this->statsd->increment($metric . $tags); - } - if($metric == $usersMetrics[0]) { - $this->statsd->increment('users.{scope}.count.total' . $tags); - } - if($metric == $usersMetrics[3]) { - $this->statsd->decrement('users.{scope}.count.total' . $tags); + if($metric == $usersMetrics[0]) { + $this->statsd->increment('users.{scope}.count.total' . $tags); + } + if($metric == $usersMetrics[3]) { + $this->statsd->decrement('users.{scope}.count.total' . $tags); + } } } From 5e6b8ec50695197aae53e791cdea32c0673e7dc1 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Fri, 23 Sep 2022 05:51:26 +0000 Subject: [PATCH 05/69] refactor counters to get sum in period only --- src/Appwrite/Usage/Calculators/TimeSeries.php | 181 ++++-------------- 1 file changed, 37 insertions(+), 144 deletions(-) diff --git a/src/Appwrite/Usage/Calculators/TimeSeries.php b/src/Appwrite/Usage/Calculators/TimeSeries.php index 54aff2d9f..ebe49aca6 100644 --- a/src/Appwrite/Usage/Calculators/TimeSeries.php +++ b/src/Appwrite/Usage/Calculators/TimeSeries.php @@ -14,7 +14,20 @@ class TimeSeries extends Calculator protected Database $database; protected $errorHandler; private array $latestTime = []; - + protected array $periods = [ + [ + 'key' => '1h', + 'startTime' => '-24 hours' + ], + [ + 'key' => '1d', + 'startTime' => '-30 days' + ], + [ + 'key' => '1mo', + 'startTime' => '-12 months' + ] + ]; // all the mertics that we are collecting protected array $metrics = [ 'project.$all.network.requests' => [ @@ -271,19 +284,28 @@ class TimeSeries extends Calculator 'table' => 'appwrite_usage_project_{scope}_compute_time', 'groupBy' => ['functionId'], ], - ]; - - private array $counterMetrics = [ + + // counters 'users.$all.count.total' => [ 'table' => 'appwrite_usage_users_{scope}_count_total', ], + 'buckets.$all.count.total' => [ + 'table' => 'appwrite_usage_buckets_{scope}_count_total', + ], + 'files.$all.count.total' => [ + 'table' => 'appwrite_usage_files_{scope}_count_total', + ], + 'files.bucketId.count.total' => [ + 'table' => 'appwrite_usage_files_{scope}_count_total', + 'groupBy' => ['bucketId'] + ], 'databases.$all.count.total' => [ 'table' => 'appwrite_usage_databases_{scope}_count_total', ], 'collections.$all.count.total' => [ 'table' => 'appwrite_usage_collections_{scope}_count_total', ], - 'documents.$all.count.total' => [ + 'documents.databaseId.count.total' => [ 'table' => 'appwrite_usage_documents_{scope}_count_total', 'groupBy' => ['databaseId'] ], @@ -301,11 +323,6 @@ class TimeSeries extends Calculator ], ]; - protected array $period = [ - 'key' => '1h', - 'startTime' => '-24 hours', - ]; - public function __construct(Database $database, InfluxDatabase $influxDB, callable $errorHandler = null) { $this->database = $database; @@ -359,121 +376,6 @@ class TimeSeries extends Calculator } } } - /** - * Create or Update Count Mertic - * Create or update each metric in the stats collection for the given project - * - * @param string $projectId - * @param int $time - * @param string $period - * @param string $metric - * @param int $value - * @param int $type - * - * @return void - */ - private function createOrUpdateCountMetric(string $projectId, string $time, string $period, string $metric, int $value): void - { - $id = \md5("{$time}_{$period}_{$metric}"); - $this->database->setNamespace('_console'); - $project = $this->database->getDocument('projects', $projectId); - $this->database->setNamespace('_' . $project->getInternalId()); - - try { - $document = $this->database->getDocument('stats', $id); - if ($document->isEmpty()) { - $this->database->createDocument('stats', new Document([ - '$id' => $id, - 'period' => $period, - 'time' => $time, - 'metric' => $metric, - 'value' => $value, - 'type' => 1, - ])); - } else { - $current = $document->getAttribute('value', 0); - $value = $current + $value; - $this->database->updateDocument( - 'stats', - $document->getId(), - $document->setAttribute('value', $value) - ); - } - } catch (\Exception $e) { // if projects are deleted this might fail - if (is_callable($this->errorHandler)) { - call_user_func($this->errorHandler, $e, "sync_project_{$projectId}_metric_{$metric}"); - } else { - throw $e; - } - } - } - - private function syncCounters(string $metric, array $options, array $period) { - $start = null; // need to calculate last updated date - if (!empty($this->latestTime[$metric][$period['key']])) { - $start = $this->latestTime[$metric][$period['key']]; - } - $end = (new DateTime())->format(DateTime::RFC3339); - - $table = $options['table']; - $groupBy = empty($options['groupBy']) ? '' : ', ' . implode(', ', array_map(fn($groupBy) => '"' . $groupBy . '" ', $options['groupBy'])); //Some sub level metrics may be grouped by other tags like collectionId, bucketId, etc - - $filters = $options['filters'] ?? []; // Some metrics might have additional filters, like function's status - if (!empty($filters)) { - $filters = ' AND ' . implode(' AND ', array_map(fn ($filter, $value) => "\"{$filter}\"='{$value}'", array_keys($filters), array_values($filters))); - } else { - $filters = ''; - } - - $query = "SELECT sum(value) AS \"value\" "; - $query .= "FROM \"{$table}\" "; - $query .= "WHERE "; - $query .= is_null($start) ? '' : "\"time\" > '{$start}' "; - $query .= is_null($start) ? '' : "AND "; - $query .= "\"time\" < '{$end}' "; - $query .= "AND \"metric_type\"='counter' {$filters} "; - $query .= "GROUP BY time({$period['key']}), \"projectId\" {$groupBy} "; - $query .= "FILL(null)"; - - try { - $result = $this->influxDB->query($query); - $points = $result->getPoints(); - foreach ($points as $point) { - $projectId = $point['projectId']; - - if (!empty($projectId) && $projectId !== 'console') { - $metricUpdated = $metric; - if (!empty($groupBy)) { - foreach ($options['groupBy'] as $groupBy) { - $groupedBy = $point[$groupBy] ?? ''; - if (empty($groupedBy)) { - continue; - } - $metricUpdated = str_replace($groupBy, $groupedBy, $metricUpdated); - } - } - - $value = (!empty($point['value'])) ? $point['value'] : 0; - - $this->createOrUpdateCountMetric( - $projectId, - $point['time'], - $period['key'], - $metricUpdated, - $value, - 0 - ); - $this->latestTime[$metric][$period['key']] = $point['time']; - } - } - } catch (\Exception $e) { // if projects are deleted this might fail - if (is_callable($this->errorHandler)) { - call_user_func($this->errorHandler, $e, "sync_metric_{$metric}_influxdb"); - } else { - throw $e; - } - } - } /** * Sync From InfluxDB @@ -559,28 +461,19 @@ class TimeSeries extends Calculator */ public function collect(): void { - foreach($this->counterMetrics as $metric => $options) { - try { - $this->syncCounters($metric, $options, $this->period); - } catch (\Exception $e) { - if (is_callable($this->errorHandler)) { - call_user_func($this->errorHandler, $e); - } else { - throw $e; + foreach ($this->periods as $period) { + foreach ($this->metrics as $metric => $options) { //for each metrics + try { + $this->syncFromInfluxDB($metric, $options, $period); + } catch (\Exception $e) { + if (is_callable($this->errorHandler)) { + call_user_func($this->errorHandler, $e); + } else { + throw $e; + } } } } - foreach ($this->metrics as $metric => $options) { //for each metrics - try { - $this->syncFromInfluxDB($metric, $options, $this->period); - } catch (\Exception $e) { - if (is_callable($this->errorHandler)) { - call_user_func($this->errorHandler, $e); - } else { - throw $e; - } - } - } } } From 4969ceadd4e73f8565dfadecef435e6f79dc0581 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 25 Sep 2022 02:07:26 +0000 Subject: [PATCH 06/69] add project internal id to the timeseries tats --- app/controllers/shared/api.php | 1 + src/Appwrite/Usage/Stats.php | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 34ab14858..4959c2fe2 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -149,6 +149,7 @@ App::init() ->setUser($user); $usage + ->setParam('projectInternalId', $project->getInternalId()) ->setParam('projectId', $project->getId()) ->setParam('project.{scope}.network.requests', 1) ->setParam('httpMethod', $request->getMethod()) diff --git a/src/Appwrite/Usage/Stats.php b/src/Appwrite/Usage/Stats.php index e4d5640ea..bdd2e8498 100644 --- a/src/Appwrite/Usage/Stats.php +++ b/src/Appwrite/Usage/Stats.php @@ -80,7 +80,8 @@ class Stats public function submit(): void { $projectId = $this->params['projectId'] ?? ''; - $tags = ",projectId={$projectId},version=" . App::getEnv('_APP_VERSION', 'UNKNOWN'); + $projectInternalId = $this->params['projectInternalId']; + $tags = ",projectInternalId={$projectInternalId},projectId={$projectId},version=" . App::getEnv('_APP_VERSION', 'UNKNOWN'); // the global namespace is prepended to every key (optional) $this->statsd->setNamespace($this->namespace); From 895e12c29bbcad9d81ff3d16344168ec968e7597 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 25 Sep 2022 02:08:42 +0000 Subject: [PATCH 07/69] monthly period not supported --- src/Appwrite/Usage/Calculators/TimeSeries.php | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/Appwrite/Usage/Calculators/TimeSeries.php b/src/Appwrite/Usage/Calculators/TimeSeries.php index ebe49aca6..5d1e78533 100644 --- a/src/Appwrite/Usage/Calculators/TimeSeries.php +++ b/src/Appwrite/Usage/Calculators/TimeSeries.php @@ -22,10 +22,6 @@ class TimeSeries extends Calculator [ 'key' => '1d', 'startTime' => '-30 days' - ], - [ - 'key' => '1mo', - 'startTime' => '-12 months' ] ]; // all the mertics that we are collecting From 22b00c019d63ef66acd635f0c0e1847bb3a3f04a Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 9 Oct 2022 06:30:26 +0000 Subject: [PATCH 08/69] update usage period on get for databases --- app/controllers/api/databases.php | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index e2acb3077..657055f28 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -2467,8 +2467,8 @@ App::get('/v1/databases/usage') if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { $periods = [ '24h' => [ - 'period' => '30m', - 'limit' => 48, + 'period' => '1h', + 'limit' => 24, ], '7d' => [ 'period' => '1d', @@ -2586,8 +2586,8 @@ App::get('/v1/databases/:databaseId/usage') if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { $periods = [ '24h' => [ - 'period' => '30m', - 'limit' => 48, + 'period' => '1h', + 'limit' => 24, ], '7d' => [ 'period' => '1d', @@ -2706,8 +2706,8 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/usage') if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { $periods = [ '24h' => [ - 'period' => '30m', - 'limit' => 48, + 'period' => '1h', + 'limit' => 24, ], '7d' => [ 'period' => '1d', From 9b2db20b3b1661bde68934d14a0ea73fa450d23e Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 9 Oct 2022 06:33:14 +0000 Subject: [PATCH 09/69] fix backfill --- app/controllers/api/databases.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index 657055f28..513c9f265 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -2529,7 +2529,7 @@ App::get('/v1/databases/usage') while ($backfill > 0) { $last = $limit - $backfill - 1; // array index of last added metric $diff = match ($period) { // convert period to seconds for unix timestamp math - '30m' => 1800, + '1h' => 3600, '1d' => 86400, }; $stats[$metric][] = [ @@ -2643,7 +2643,7 @@ App::get('/v1/databases/:databaseId/usage') while ($backfill > 0) { $last = $limit - $backfill - 1; // array index of last added metric $diff = match ($period) { // convert period to seconds for unix timestamp math - '30m' => 1800, + '1h' => 3600, '1d' => 86400, }; $stats[$metric][] = [ @@ -2758,7 +2758,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/usage') while ($backfill > 0) { $last = $limit - $backfill - 1; // array index of last added metric $diff = match ($period) { // convert period to seconds for unix timestamp math - '30m' => 1800, + '1h' => 3600, '1d' => 86400, }; $stats[$metric][] = [ From 8ddbde62c1eb7b6bfefcb5dfb703e1890a8c3874 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 9 Oct 2022 06:45:46 +0000 Subject: [PATCH 10/69] use project internal id --- src/Appwrite/Usage/Calculators/TimeSeries.php | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/Appwrite/Usage/Calculators/TimeSeries.php b/src/Appwrite/Usage/Calculators/TimeSeries.php index 5d1e78533..8d5f0f0d7 100644 --- a/src/Appwrite/Usage/Calculators/TimeSeries.php +++ b/src/Appwrite/Usage/Calculators/TimeSeries.php @@ -342,9 +342,7 @@ class TimeSeries extends Calculator private function createOrUpdateMetric(string $projectId, string $time, string $period, string $metric, int $value, int $type): void { $id = \md5("{$time}_{$period}_{$metric}"); - $this->database->setNamespace('_console'); - $project = $this->database->getDocument('projects', $projectId); - $this->database->setNamespace('_' . $project->getInternalId()); + $this->database->setNamespace('_' . $projectId); try { $document = $this->database->getDocument('stats', $id); @@ -430,7 +428,7 @@ class TimeSeries extends Calculator $value = (!empty($point['value'])) ? $point['value'] : 0; $this->createOrUpdateMetric( - $projectId, + $point['projectInternalId'], $point['time'], $period['key'], $metricUpdated, From f41d388bf50669b78a562b9c6bfbb31065a24e2b Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 10 Oct 2022 09:13:20 +0000 Subject: [PATCH 11/69] separate cumulative metrics --- src/Appwrite/Usage/Calculators/TimeSeries.php | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/src/Appwrite/Usage/Calculators/TimeSeries.php b/src/Appwrite/Usage/Calculators/TimeSeries.php index 8d5f0f0d7..dc77f9409 100644 --- a/src/Appwrite/Usage/Calculators/TimeSeries.php +++ b/src/Appwrite/Usage/Calculators/TimeSeries.php @@ -319,6 +319,22 @@ class TimeSeries extends Calculator ], ]; + protected array $cumulativeMetrics = [ + 'deployments.$all.storage.size' => [ + 'table' => 'appwrite_usage_deployments_{scope}_storage_size', + ], + 'project.$all.storage.size' => [ + 'table' => 'appwrite_usage_project_{scope}_storage_size', + ], + 'files.$all.storage.size' => [ + 'table' => 'appwrite_usage_files_{scope}_storage_size', + ], + 'files.$bucketId.storage.size' => [ + 'table' => 'appwrite_usage_files_{scope}_storage_size', + 'groupBy' => ['bucketId'] + ], + ]; + public function __construct(Database $database, InfluxDatabase $influxDB, callable $errorHandler = null) { $this->database = $database; @@ -469,5 +485,18 @@ class TimeSeries extends Calculator } } + // for cumulative metrics only get hourly metrics from timeseries + foreach ($this->cumulativeMetrics as $metric => $options) { + try { + $this->syncFromInfluxDB($metric, $options, $this->periods[0]); + } catch (\Exception $e) { + if (is_callable($this->errorHandler)) { + call_user_func($this->errorHandler, $e); + } else { + throw $e; + } + } + } + } } From 3ffc31ba58fb28914f477f0de32d35f433d05631 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 10 Oct 2022 09:39:26 +0000 Subject: [PATCH 12/69] fix formatting --- src/Appwrite/Usage/Calculators/TimeSeries.php | 3 +-- src/Appwrite/Usage/Stats.php | 24 +++++++++---------- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/src/Appwrite/Usage/Calculators/TimeSeries.php b/src/Appwrite/Usage/Calculators/TimeSeries.php index dc77f9409..f75882bd3 100644 --- a/src/Appwrite/Usage/Calculators/TimeSeries.php +++ b/src/Appwrite/Usage/Calculators/TimeSeries.php @@ -280,7 +280,7 @@ class TimeSeries extends Calculator 'table' => 'appwrite_usage_project_{scope}_compute_time', 'groupBy' => ['functionId'], ], - + // counters 'users.$all.count.total' => [ 'table' => 'appwrite_usage_users_{scope}_count_total', @@ -497,6 +497,5 @@ class TimeSeries extends Calculator } } } - } } diff --git a/src/Appwrite/Usage/Stats.php b/src/Appwrite/Usage/Stats.php index bdd2e8498..ea77560bb 100644 --- a/src/Appwrite/Usage/Stats.php +++ b/src/Appwrite/Usage/Stats.php @@ -109,10 +109,10 @@ class Stats $value = $this->params[$metric] ?? 0; if ($value >= 1) { $this->statsd->increment($metric . $tags); - if($metric == $usersMetrics[0]) { + if ($metric == $usersMetrics[0]) { $this->statsd->increment('users.{scope}.count.total' . $tags); } - if($metric == $usersMetrics[3]) { + if ($metric == $usersMetrics[3]) { $this->statsd->decrement('users.{scope}.count.total' . $tags); } } @@ -139,24 +139,24 @@ class Stats $dbTags = $tags . ",collectionId=" . ($this->params['collectionId'] ?? '') . ",databaseId=" . ($this->params['databaseId'] ?? ''); $this->statsd->increment($metric . $dbTags); - if($metric == $dbMetrics[0]) { + if ($metric == $dbMetrics[0]) { $this->statsd->increment('databases.{scope}.count.total' . $dbTags); } - if($metric == $dbMetrics[3]) { + if ($metric == $dbMetrics[3]) { $this->statsd->decrement('databases.{scope}.count.total' . $dbTags); } - if($metric == $dbMetrics[4]) { + if ($metric == $dbMetrics[4]) { $this->statsd->increment('collections.{scope}.count.total' . $dbTags); } - if($metric == $dbMetrics[7]) { + if ($metric == $dbMetrics[7]) { $this->statsd->decrement('collections.{scope}.count.total' . $dbTags); } - if($metric == $dbMetrics[8]) { + if ($metric == $dbMetrics[8]) { $this->statsd->increment('documents.{scope}.count.total' . $dbTags); } - if($metric == $dbMetrics[11]) { + if ($metric == $dbMetrics[11]) { $this->statsd->decrement('documents.{scope}.count.total' . $dbTags); } } @@ -179,17 +179,17 @@ class Stats $storageTags = $tags . ",bucketId=" . ($this->params['bucketId'] ?? ''); $this->statsd->increment($metric . $storageTags); - if($metric == $storageMertics[0]) { + if ($metric == $storageMertics[0]) { $this->statsd->increment('buckets.{scope}.count.total' . $storageTags); } - if($metric == $storageMertics[3]) { + if ($metric == $storageMertics[3]) { $this->statsd->decrement('buckets.{scope}.count.total' . $storageTags); } - if($metric == $storageMertics[4]) { + if ($metric == $storageMertics[4]) { $this->statsd->increment('files.{scope}.count.total' . $storageTags); } - if($metric == $storageMertics[7]) { + if ($metric == $storageMertics[7]) { $this->statsd->decrement('files.{scope}.count.total' . $storageTags); } } From 910c033d2a6887663b5bd095e3f889fa7b05795b Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 11 Oct 2022 05:40:28 +0000 Subject: [PATCH 13/69] refactor build and execution time --- src/Appwrite/Usage/Calculators/TimeSeries.php | 41 ++++++++++--------- 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/src/Appwrite/Usage/Calculators/TimeSeries.php b/src/Appwrite/Usage/Calculators/TimeSeries.php index f75882bd3..7c80e41e6 100644 --- a/src/Appwrite/Usage/Calculators/TimeSeries.php +++ b/src/Appwrite/Usage/Calculators/TimeSeries.php @@ -198,12 +198,6 @@ class TimeSeries extends Calculator 'executions.$all.compute.total' => [ 'table' => 'appwrite_usage_executions_{scope}_compute', ], - 'builds.$all.compute.time' => [ - 'table' => 'appwrite_usage_executions_{scope}_compute_time', - ], - 'executions.$all.compute.time' => [ - 'table' => 'appwrite_usage_executions_{scope}_compute_time', - ], 'builds.$all.compute.total' => [ 'table' => 'appwrite_usage_builds_{scope}_compute', ], @@ -239,14 +233,7 @@ class TimeSeries extends Calculator 'table' => 'appwrite_usage_builds_{scope}_compute', 'groupBy' => ['functionId'], ], - 'executions.functionId.compute.time' => [ - 'table' => 'appwrite_usage_executions_{scope}_compute_time', - 'groupBy' => ['functionId'], - ], - 'builds.functionId.compute.time' => [ - 'table' => 'appwrite_usage_builds_{scope}_compute_time', - 'groupBy' => ['functionId'], - ], + 'executions.functionId.compute.failure' => [ 'table' => 'appwrite_usage_executions_{scope}_compute', 'groupBy' => ['functionId'], @@ -276,11 +263,6 @@ class TimeSeries extends Calculator ], ], - 'project.$all.compute.time' => [ // Built time + execution time - 'table' => 'appwrite_usage_project_{scope}_compute_time', - 'groupBy' => ['functionId'], - ], - // counters 'users.$all.count.total' => [ 'table' => 'appwrite_usage_users_{scope}_count_total', @@ -333,6 +315,27 @@ class TimeSeries extends Calculator 'table' => 'appwrite_usage_files_{scope}_storage_size', 'groupBy' => ['bucketId'] ], + + 'builds.$all.compute.time' => [ + 'table' => 'appwrite_usage_executions_{scope}_compute_time', + ], + 'executions.$all.compute.time' => [ + 'table' => 'appwrite_usage_executions_{scope}_compute_time', + ], + + 'executions.functionId.compute.time' => [ + 'table' => 'appwrite_usage_executions_{scope}_compute_time', + 'groupBy' => ['functionId'], + ], + 'builds.functionId.compute.time' => [ + 'table' => 'appwrite_usage_builds_{scope}_compute_time', + 'groupBy' => ['functionId'], + ], + + 'project.$all.compute.time' => [ // Built time + execution time + 'table' => 'appwrite_usage_project_{scope}_compute_time', + 'groupBy' => ['functionId'], + ], ]; public function __construct(Database $database, InfluxDatabase $influxDB, callable $errorHandler = null) From e7e8c53d89ff655581c849a930c2c94c78124163 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Fri, 14 Oct 2022 08:49:00 +0000 Subject: [PATCH 14/69] use same array for all metrics --- src/Appwrite/Usage/Calculators/TimeSeries.php | 22 +++---------------- 1 file changed, 3 insertions(+), 19 deletions(-) diff --git a/src/Appwrite/Usage/Calculators/TimeSeries.php b/src/Appwrite/Usage/Calculators/TimeSeries.php index 7c80e41e6..d03a831e3 100644 --- a/src/Appwrite/Usage/Calculators/TimeSeries.php +++ b/src/Appwrite/Usage/Calculators/TimeSeries.php @@ -299,9 +299,6 @@ class TimeSeries extends Calculator 'table' => 'appwrite_usage_documents_{scope}_count_total', 'groupBy' => ['databaseId', 'collectionId'] ], - ]; - - protected array $cumulativeMetrics = [ 'deployments.$all.storage.size' => [ 'table' => 'appwrite_usage_deployments_{scope}_storage_size', ], @@ -315,14 +312,14 @@ class TimeSeries extends Calculator 'table' => 'appwrite_usage_files_{scope}_storage_size', 'groupBy' => ['bucketId'] ], - + 'builds.$all.compute.time' => [ 'table' => 'appwrite_usage_executions_{scope}_compute_time', ], 'executions.$all.compute.time' => [ 'table' => 'appwrite_usage_executions_{scope}_compute_time', ], - + 'executions.functionId.compute.time' => [ 'table' => 'appwrite_usage_executions_{scope}_compute_time', 'groupBy' => ['functionId'], @@ -331,7 +328,7 @@ class TimeSeries extends Calculator 'table' => 'appwrite_usage_builds_{scope}_compute_time', 'groupBy' => ['functionId'], ], - + 'project.$all.compute.time' => [ // Built time + execution time 'table' => 'appwrite_usage_project_{scope}_compute_time', 'groupBy' => ['functionId'], @@ -487,18 +484,5 @@ class TimeSeries extends Calculator } } } - - // for cumulative metrics only get hourly metrics from timeseries - foreach ($this->cumulativeMetrics as $metric => $options) { - try { - $this->syncFromInfluxDB($metric, $options, $this->periods[0]); - } catch (\Exception $e) { - if (is_callable($this->errorHandler)) { - call_user_func($this->errorHandler, $e); - } else { - throw $e; - } - } - } } } From 0316710ee4e831937740177546f27161e83fab27 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Fri, 14 Oct 2022 08:51:37 +0000 Subject: [PATCH 15/69] update database --- composer.json | 2 +- composer.lock | 53 ++++++++++++++++++++++++++++++--------------------- 2 files changed, 32 insertions(+), 23 deletions(-) diff --git a/composer.json b/composer.json index cad6b65fd..219b4d059 100644 --- a/composer.json +++ b/composer.json @@ -51,7 +51,7 @@ "utopia-php/cache": "0.6.*", "utopia-php/cli": "0.13.*", "utopia-php/config": "0.2.*", - "utopia-php/database": "0.25.*", + "utopia-php/database": "dev-feat-events-and-listeners as 0.25.0", "utopia-php/locale": "0.4.*", "utopia-php/registry": "0.5.*", "utopia-php/preloader": "0.2.*", diff --git a/composer.lock b/composer.lock index 532e73ddb..305b1995f 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "fca1e9492b5c12b6dbf6607418ca8ec8", + "content-hash": "666c02369c446a1c5ca77a3de8d98988", "packages": [ { "name": "adhocore/jwt", @@ -2060,16 +2060,16 @@ }, { "name": "utopia-php/database", - "version": "0.25.4", + "version": "dev-feat-events-and-listeners", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "2883de82eee99e5744bf6e4123095a530c48a194" + "reference": "5af85043c88b6b3f087f444d45325c30adeb84c2" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/2883de82eee99e5744bf6e4123095a530c48a194", - "reference": "2883de82eee99e5744bf6e4123095a530c48a194", + "url": "https://api.github.com/repos/utopia-php/database/zipball/5af85043c88b6b3f087f444d45325c30adeb84c2", + "reference": "5af85043c88b6b3f087f444d45325c30adeb84c2", "shasum": "" }, "require": { @@ -2118,9 +2118,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/0.25.4" + "source": "https://github.com/utopia-php/database/tree/feat-events-and-listeners" }, - "time": "2022-09-14T06:22:33+00:00" + "time": "2022-10-14T07:31:42+00:00" }, { "name": "utopia-php/domains", @@ -4124,16 +4124,16 @@ }, { "name": "sebastian/comparator", - "version": "4.0.7", + "version": "4.0.8", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/comparator.git", - "reference": "7fa545db548c90bdebeb9da0583001a252be5578" + "reference": "fa0f136dd2334583309d32b62544682ee972b51a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/comparator/zipball/7fa545db548c90bdebeb9da0583001a252be5578", - "reference": "7fa545db548c90bdebeb9da0583001a252be5578", + "url": "https://api.github.com/repos/sebastianbergmann/comparator/zipball/fa0f136dd2334583309d32b62544682ee972b51a", + "reference": "fa0f136dd2334583309d32b62544682ee972b51a", "shasum": "" }, "require": { @@ -4186,7 +4186,7 @@ ], "support": { "issues": "https://github.com/sebastianbergmann/comparator/issues", - "source": "https://github.com/sebastianbergmann/comparator/tree/4.0.7" + "source": "https://github.com/sebastianbergmann/comparator/tree/4.0.8" }, "funding": [ { @@ -4194,7 +4194,7 @@ "type": "github" } ], - "time": "2022-09-14T06:33:43+00:00" + "time": "2022-09-14T12:41:17+00:00" }, { "name": "sebastian/complexity", @@ -5283,16 +5283,16 @@ }, { "name": "twig/twig", - "version": "v3.4.2", + "version": "v3.4.3", "source": { "type": "git", "url": "https://github.com/twigphp/Twig.git", - "reference": "e07cdd3d430cd7e453c31b36eb5ad6c0c5e43077" + "reference": "c38fd6b0b7f370c198db91ffd02e23b517426b58" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/twigphp/Twig/zipball/e07cdd3d430cd7e453c31b36eb5ad6c0c5e43077", - "reference": "e07cdd3d430cd7e453c31b36eb5ad6c0c5e43077", + "url": "https://api.github.com/repos/twigphp/Twig/zipball/c38fd6b0b7f370c198db91ffd02e23b517426b58", + "reference": "c38fd6b0b7f370c198db91ffd02e23b517426b58", "shasum": "" }, "require": { @@ -5343,7 +5343,7 @@ ], "support": { "issues": "https://github.com/twigphp/Twig/issues", - "source": "https://github.com/twigphp/Twig/tree/v3.4.2" + "source": "https://github.com/twigphp/Twig/tree/v3.4.3" }, "funding": [ { @@ -5355,12 +5355,21 @@ "type": "tidelift" } ], - "time": "2022-08-12T06:47:24+00:00" + "time": "2022-09-28T08:42:51+00:00" + } + ], + "aliases": [ + { + "package": "utopia-php/database", + "version": "dev-feat-events-and-listeners", + "alias": "0.25.0", + "alias_normalized": "0.25.0.0" } ], - "aliases": [], "minimum-stability": "stable", - "stability-flags": [], + "stability-flags": { + "utopia-php/database": 20 + }, "prefer-stable": false, "prefer-lowest": false, "platform": { @@ -5384,5 +5393,5 @@ "platform-overrides": { "php": "8.0" }, - "plugin-api-version": "2.3.0" + "plugin-api-version": "2.2.0" } From dee4f5c5177403acd6a8e3a897e30050704003c1 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Fri, 14 Oct 2022 11:07:07 +0000 Subject: [PATCH 16/69] use metadata constants --- src/Appwrite/Resque/Worker.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Resque/Worker.php b/src/Appwrite/Resque/Worker.php index 40adc3e52..dd7cebd08 100644 --- a/src/Appwrite/Resque/Worker.php +++ b/src/Appwrite/Resque/Worker.php @@ -229,7 +229,7 @@ abstract class Worker throw new \Exception("Project does not exist: {$projectId}"); } - if ($type === self::DATABASE_CONSOLE && !$database->exists($database->getDefaultDatabase(), '_metadata')) { + if ($type === self::DATABASE_CONSOLE && !$database->exists($database->getDefaultDatabase(), Database::METADATA)) { throw new \Exception('Console project not ready'); } From e7b9ba70b626a0eac5b18844afc42ddd0c185b10 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Fri, 14 Oct 2022 11:07:20 +0000 Subject: [PATCH 17/69] fix error --- src/Appwrite/Auth/Auth.php | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/Appwrite/Auth/Auth.php b/src/Appwrite/Auth/Auth.php index f4cea0166..727f6ecf7 100644 --- a/src/Appwrite/Auth/Auth.php +++ b/src/Appwrite/Auth/Auth.php @@ -14,6 +14,7 @@ use Utopia\Database\Document; use Utopia\Database\DateTime; use Utopia\Database\Role; use Utopia\Database\Validator\Authorization; +use Utopia\Database\Validator\Roles; class Auth { @@ -427,11 +428,11 @@ class Auth $phoneVerified = $user->getAttribute('phoneVerification', false); if ($emailVerified || $phoneVerified) { - $roles[] = Role::user($user->getId(), Database::DIMENSION_VERIFIED)->toString(); - $roles[] = Role::users(Database::DIMENSION_VERIFIED)->toString(); + $roles[] = Role::user($user->getId(), Roles::DIMENSION_VERIFIED)->toString(); + $roles[] = Role::users(Roles::DIMENSION_VERIFIED)->toString(); } else { - $roles[] = Role::user($user->getId(), Database::DIMENSION_UNVERIFIED)->toString(); - $roles[] = Role::users(Database::DIMENSION_UNVERIFIED)->toString(); + $roles[] = Role::user($user->getId(), Roles::DIMENSION_UNVERIFIED)->toString(); + $roles[] = Role::users(Roles::DIMENSION_UNVERIFIED)->toString(); } } else { return [Role::guests()->toString()]; From 65337bc9e93a80ebd463022635a5796498742557 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Fri, 14 Oct 2022 11:07:32 +0000 Subject: [PATCH 18/69] fix metadata creation --- app/http.php | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/app/http.php b/app/http.php index afda7053b..5a7ab93af 100644 --- a/app/http.php +++ b/app/http.php @@ -91,20 +91,13 @@ $http->on('start', function (Server $http) use ($payloadSize, $register) { /** @var array $collections */ $collections = Config::getParam('collections', []); - if (!$dbForConsole->exists(App::getEnv('_APP_DB_SCHEMA', 'appwrite'))) { - $redis->flushAll(); + $redis->flushAll(); - Console::success('[Setup] - Creating database: appwrite...'); + Console::success('[Setup] - Creating database: appwrite...'); - $dbForConsole->create(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); - } + $dbForConsole->create(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); - try { - Console::success('[Setup] - Creating metadata table: appwrite...'); - $dbForConsole->createMetadata(); - } catch (\Throwable $th) { - Console::success('[Setup] - Skip: metadata table already exists'); - } + Console::success('[Setup] - Created database: appwrite and metadata tables...'); if ($dbForConsole->getCollection(Audit::COLLECTION)->isEmpty()) { $audit = new Audit($dbForConsole); From 83071d85e37110675e6646d47fe1e41cb5b121eb Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Fri, 14 Oct 2022 11:10:06 +0000 Subject: [PATCH 19/69] fix test constant --- tests/unit/Auth/AuthTest.php | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/tests/unit/Auth/AuthTest.php b/tests/unit/Auth/AuthTest.php index 43a2bef8e..822b612db 100644 --- a/tests/unit/Auth/AuthTest.php +++ b/tests/unit/Auth/AuthTest.php @@ -10,6 +10,7 @@ use Utopia\Database\Role; use Utopia\Database\Validator\Authorization; use PHPUnit\Framework\TestCase; use Utopia\Database\Database; +use Utopia\Database\Validator\Roles; class AuthTest extends TestCase { @@ -379,8 +380,8 @@ class AuthTest extends TestCase $this->assertCount(11, $roles); $this->assertContains(Role::users()->toString(), $roles); $this->assertContains(Role::user(ID::custom('123'))->toString(), $roles); - $this->assertContains(Role::users(Database::DIMENSION_VERIFIED)->toString(), $roles); - $this->assertContains(Role::user(ID::custom('123'), Database::DIMENSION_VERIFIED)->toString(), $roles); + $this->assertContains(Role::users(Roles::DIMENSION_VERIFIED)->toString(), $roles); + $this->assertContains(Role::user(ID::custom('123'), Roles::DIMENSION_VERIFIED)->toString(), $roles); $this->assertContains(Role::team(ID::custom('abc'))->toString(), $roles); $this->assertContains(Role::team(ID::custom('abc'), 'administrator')->toString(), $roles); $this->assertContains(Role::team(ID::custom('abc'), 'moderator')->toString(), $roles); @@ -394,15 +395,15 @@ class AuthTest extends TestCase $user['phoneVerification'] = false; $roles = Auth::getRoles($user); - $this->assertContains(Role::users(Database::DIMENSION_UNVERIFIED)->toString(), $roles); - $this->assertContains(Role::user(ID::custom('123'), Database::DIMENSION_UNVERIFIED)->toString(), $roles); + $this->assertContains(Role::users(Roles::DIMENSION_UNVERIFIED)->toString(), $roles); + $this->assertContains(Role::user(ID::custom('123'), Roles::DIMENSION_UNVERIFIED)->toString(), $roles); // Enable single verification type $user['emailVerification'] = true; $roles = Auth::getRoles($user); - $this->assertContains(Role::users(Database::DIMENSION_VERIFIED)->toString(), $roles); - $this->assertContains(Role::user(ID::custom('123'), Database::DIMENSION_VERIFIED)->toString(), $roles); + $this->assertContains(Role::users(Roles::DIMENSION_VERIFIED)->toString(), $roles); + $this->assertContains(Role::user(ID::custom('123'), Roles::DIMENSION_VERIFIED)->toString(), $roles); } public function testPrivilegedUserRoles(): void @@ -438,8 +439,8 @@ class AuthTest extends TestCase $this->assertCount(7, $roles); $this->assertNotContains(Role::users()->toString(), $roles); $this->assertNotContains(Role::user(ID::custom('123'))->toString(), $roles); - $this->assertNotContains(Role::users(Database::DIMENSION_VERIFIED)->toString(), $roles); - $this->assertNotContains(Role::user(ID::custom('123'), Database::DIMENSION_VERIFIED)->toString(), $roles); + $this->assertNotContains(Role::users(Roles::DIMENSION_VERIFIED)->toString(), $roles); + $this->assertNotContains(Role::user(ID::custom('123'), Roles::DIMENSION_VERIFIED)->toString(), $roles); $this->assertContains(Role::team(ID::custom('abc'))->toString(), $roles); $this->assertContains(Role::team(ID::custom('abc'), 'administrator')->toString(), $roles); $this->assertContains(Role::team(ID::custom('abc'), 'moderator')->toString(), $roles); From f86a5b5fcd7d8528e21c3ba16222a02dc3f08926 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Fri, 14 Oct 2022 11:12:59 +0000 Subject: [PATCH 20/69] fix renamed methods --- app/controllers/api/databases.php | 2 +- app/init.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index 513c9f265..d0ae1be88 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -1574,7 +1574,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/indexes') Query::equal('databaseInternalId', [$db->getInternalId()]) ], 61); - $limit = 64 - MariaDB::getNumberOfDefaultIndexes(); + $limit = 64 - MariaDB::getCountOfDefaultIndexes(); if ($count >= $limit) { throw new Exception(Exception::INDEX_LIMIT_EXCEEDED, 'Index limit exceeded'); diff --git a/app/init.php b/app/init.php index 3c9f785f9..8cb0fc170 100644 --- a/app/init.php +++ b/app/init.php @@ -282,7 +282,7 @@ Database::addFilter( ->find('attributes', [ Query::equal('collectionInternalId', [$document->getInternalId()]), Query::equal('databaseInternalId', [$document->getAttribute('databaseInternalId')]), - Query::limit($database->getAttributeLimit()), + Query::limit($database->getLimitForAttributes()), ]); } ); From b29e2c1808992a4a21e9a0efd0aa927ac283a6e9 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 16 Oct 2022 08:08:38 +0000 Subject: [PATCH 21/69] use database from master --- composer.json | 2 +- composer.lock | 48 ++++++++++++++++++++++-------------------------- 2 files changed, 23 insertions(+), 27 deletions(-) diff --git a/composer.json b/composer.json index 219b4d059..2b0373541 100644 --- a/composer.json +++ b/composer.json @@ -51,7 +51,7 @@ "utopia-php/cache": "0.6.*", "utopia-php/cli": "0.13.*", "utopia-php/config": "0.2.*", - "utopia-php/database": "dev-feat-events-and-listeners as 0.25.0", + "utopia-php/database": "dev-main as 0.25.0", "utopia-php/locale": "0.4.*", "utopia-php/registry": "0.5.*", "utopia-php/preloader": "0.2.*", diff --git a/composer.lock b/composer.lock index 305b1995f..45f6a0df2 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "666c02369c446a1c5ca77a3de8d98988", + "content-hash": "a4744aca2107d437553fe879ae4ce201", "packages": [ { "name": "adhocore/jwt", @@ -2060,16 +2060,16 @@ }, { "name": "utopia-php/database", - "version": "dev-feat-events-and-listeners", + "version": "dev-main", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "5af85043c88b6b3f087f444d45325c30adeb84c2" + "reference": "0644cd8a7efee2fc745ed70dbf2b73a78eeefa52" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/5af85043c88b6b3f087f444d45325c30adeb84c2", - "reference": "5af85043c88b6b3f087f444d45325c30adeb84c2", + "url": "https://api.github.com/repos/utopia-php/database/zipball/0644cd8a7efee2fc745ed70dbf2b73a78eeefa52", + "reference": "0644cd8a7efee2fc745ed70dbf2b73a78eeefa52", "shasum": "" }, "require": { @@ -2088,6 +2088,7 @@ "utopia-php/cli": "^0.11.0", "vimeo/psalm": "4.0.1" }, + "default-branch": true, "type": "library", "autoload": { "psr-4": { @@ -2098,16 +2099,6 @@ "license": [ "MIT" ], - "authors": [ - { - "name": "Eldad Fux", - "email": "eldad@appwrite.io" - }, - { - "name": "Brandon Leckemby", - "email": "brandon@appwrite.io" - } - ], "description": "A simple library to manage application persistency using multiple database adapters", "keywords": [ "database", @@ -2118,9 +2109,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/feat-events-and-listeners" + "source": "https://github.com/utopia-php/database/tree/0.27.0" }, - "time": "2022-10-14T07:31:42+00:00" + "time": "2022-10-16T07:39:26+00:00" }, { "name": "utopia-php/domains", @@ -3419,25 +3410,30 @@ }, { "name": "phpdocumentor/type-resolver", - "version": "1.6.1", + "version": "1.6.2", "source": { "type": "git", "url": "https://github.com/phpDocumentor/TypeResolver.git", - "reference": "77a32518733312af16a44300404e945338981de3" + "reference": "48f445a408c131e38cab1c235aa6d2bb7a0bb20d" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpDocumentor/TypeResolver/zipball/77a32518733312af16a44300404e945338981de3", - "reference": "77a32518733312af16a44300404e945338981de3", + "url": "https://api.github.com/repos/phpDocumentor/TypeResolver/zipball/48f445a408c131e38cab1c235aa6d2bb7a0bb20d", + "reference": "48f445a408c131e38cab1c235aa6d2bb7a0bb20d", "shasum": "" }, "require": { - "php": "^7.2 || ^8.0", + "php": "^7.4 || ^8.0", "phpdocumentor/reflection-common": "^2.0" }, "require-dev": { "ext-tokenizer": "*", - "psalm/phar": "^4.8" + "phpstan/extension-installer": "^1.1", + "phpstan/phpstan": "^1.8", + "phpstan/phpstan-phpunit": "^1.1", + "phpunit/phpunit": "^9.5", + "rector/rector": "^0.13.9", + "vimeo/psalm": "^4.25" }, "type": "library", "extra": { @@ -3463,9 +3459,9 @@ "description": "A PSR-5 based resolver of Class names, Types and Structural Element Names", "support": { "issues": "https://github.com/phpDocumentor/TypeResolver/issues", - "source": "https://github.com/phpDocumentor/TypeResolver/tree/1.6.1" + "source": "https://github.com/phpDocumentor/TypeResolver/tree/1.6.2" }, - "time": "2022-03-15T21:29:03+00:00" + "time": "2022-10-14T12:47:21+00:00" }, { "name": "phpspec/prophecy", @@ -5361,7 +5357,7 @@ "aliases": [ { "package": "utopia-php/database", - "version": "dev-feat-events-and-listeners", + "version": "dev-main", "alias": "0.25.0", "alias_normalized": "0.25.0.0" } From 019fa11616ecd3bd52e8ea2cf475e62507ed9493 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 16 Oct 2022 08:50:53 +0000 Subject: [PATCH 22/69] setup to use database listeners --- app/controllers/shared/api.php | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 4959c2fe2..08af6f5a3 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -16,6 +16,7 @@ use Utopia\Abuse\Abuse; use Utopia\Abuse\Adapters\TimeLimit; use Utopia\Cache\Adapter\Filesystem; use Utopia\Cache\Cache; +use Utopia\CLI\Console; use Utopia\Database\Database; use Utopia\Database\DateTime; use Utopia\Database\Document; @@ -159,6 +160,38 @@ App::init() $deletes->setProject($project); $database->setProject($project); + $dbForProject->on(Database::EVENT_DOCUMENT_CREATE, function($event, Document $document) { + $collection = $document->getCollection(); + Console::info('collection: ' . $collection); + switch($collection) + { + case 'users': + Console::info('user created'); + break; + case 'databases': + Console::info('Database created'); + break; + case 'buckets': + Console::info('bucket created'); + break; + case 'functions': + Console::info('function created'); + break; + default: + if(strpos($collection, 'buckets_') === 0) { + // created file for a bucket + Console::info('file created'); + } else if (strpos($collection, 'database_') === 0) { + if(strpos($collection, '_collection_') != false) { + Console::info('document created'); + } else { + Console::info('Collection created'); + } + } + break; + } + }); + $useCache = $route->getLabel('cache', false); if ($useCache) { From 84f5842c8100a540916e03a3bfd73a7a61451c27 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 17 Oct 2022 06:52:24 +0000 Subject: [PATCH 23/69] document create listener --- app/controllers/shared/api.php | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 08af6f5a3..acbe4466e 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -160,30 +160,31 @@ App::init() $deletes->setProject($project); $database->setProject($project); - $dbForProject->on(Database::EVENT_DOCUMENT_CREATE, function($event, Document $document) { + $dbForProject->on(Database::EVENT_DOCUMENT_CREATE, function($event, Document $document) use ($usage) { $collection = $document->getCollection(); Console::info('collection: ' . $collection); switch($collection) { case 'users': - Console::info('user created'); + $usage->setParam('users.{scope}.count.total', 1); break; case 'databases': - Console::info('Database created'); + $usage->setParam('databases.{scope}.count.total', 1); break; - case 'buckets': - Console::info('bucket created'); - break; - case 'functions': - Console::info('function created'); + case 'buckets': + $usage->setParam('buckets.{scope}.count.total', 1); break; default: if(strpos($collection, 'buckets_') === 0) { - // created file for a bucket - Console::info('file created'); + $usage + ->setParam('bucketId', $document->getAttribute('bucketId')) + ->setParam('files.{scope}.count.total', 1); } else if (strpos($collection, 'database_') === 0) { if(strpos($collection, '_collection_') != false) { - Console::info('document created'); + $usage + ->setParam('databaseId', $document->getAttribute('databaseId')) + ->setParam('collectionId', $document->getAttribute('collectionId')) + ->setParam('files.{scope}.count.total', 1); } else { Console::info('Collection created'); } From ffe029932dec0151719e2c1068814a9c986b501c Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 17 Oct 2022 06:59:27 +0000 Subject: [PATCH 24/69] refactor stats --- src/Appwrite/Usage/Stats.php | 53 ++++++------------------------------ 1 file changed, 9 insertions(+), 44 deletions(-) diff --git a/src/Appwrite/Usage/Stats.php b/src/Appwrite/Usage/Stats.php index ea77560bb..efd83a2dc 100644 --- a/src/Appwrite/Usage/Stats.php +++ b/src/Appwrite/Usage/Stats.php @@ -103,18 +103,13 @@ class Stats 'users.{scope}.requests.read', 'users.{scope}.requests.update', 'users.{scope}.requests.delete', + 'users.{scope}.count.total', ]; foreach ($usersMetrics as $metric) { $value = $this->params[$metric] ?? 0; if ($value >= 1) { - $this->statsd->increment($metric . $tags); - if ($metric == $usersMetrics[0]) { - $this->statsd->increment('users.{scope}.count.total' . $tags); - } - if ($metric == $usersMetrics[3]) { - $this->statsd->decrement('users.{scope}.count.total' . $tags); - } + $this->statsd->count($metric . $tags, $value); } } @@ -131,34 +126,16 @@ class Stats 'documents.{scope}.requests.read', 'documents.{scope}.requests.update', 'documents.{scope}.requests.delete', + 'databases.{scope}.count.total', + 'collections.{scope}.count.total', + 'documents.{scope}.count.total' ]; foreach ($dbMetrics as $metric) { $value = $this->params[$metric] ?? 0; if ($value >= 1) { $dbTags = $tags . ",collectionId=" . ($this->params['collectionId'] ?? '') . ",databaseId=" . ($this->params['databaseId'] ?? ''); - $this->statsd->increment($metric . $dbTags); - - if ($metric == $dbMetrics[0]) { - $this->statsd->increment('databases.{scope}.count.total' . $dbTags); - } - if ($metric == $dbMetrics[3]) { - $this->statsd->decrement('databases.{scope}.count.total' . $dbTags); - } - - if ($metric == $dbMetrics[4]) { - $this->statsd->increment('collections.{scope}.count.total' . $dbTags); - } - if ($metric == $dbMetrics[7]) { - $this->statsd->decrement('collections.{scope}.count.total' . $dbTags); - } - - if ($metric == $dbMetrics[8]) { - $this->statsd->increment('documents.{scope}.count.total' . $dbTags); - } - if ($metric == $dbMetrics[11]) { - $this->statsd->decrement('documents.{scope}.count.total' . $dbTags); - } + $this->statsd->count($metric . $dbTags, $value); } } @@ -171,27 +148,15 @@ class Stats 'files.{scope}.requests.read', 'files.{scope}.requests.update', 'files.{scope}.requests.delete', + 'buckets.{scope}.count.total', + 'files.{scope}.count.total' ]; foreach ($storageMertics as $metric) { $value = $this->params[$metric] ?? 0; if ($value >= 1) { $storageTags = $tags . ",bucketId=" . ($this->params['bucketId'] ?? ''); - $this->statsd->increment($metric . $storageTags); - - if ($metric == $storageMertics[0]) { - $this->statsd->increment('buckets.{scope}.count.total' . $storageTags); - } - if ($metric == $storageMertics[3]) { - $this->statsd->decrement('buckets.{scope}.count.total' . $storageTags); - } - - if ($metric == $storageMertics[4]) { - $this->statsd->increment('files.{scope}.count.total' . $storageTags); - } - if ($metric == $storageMertics[7]) { - $this->statsd->decrement('files.{scope}.count.total' . $storageTags); - } + $this->statsd->count($metric . $storageTags, $value); } } From f3c183b563b83cdb4280fb9f9658d869289064fa Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 17 Oct 2022 07:10:18 +0000 Subject: [PATCH 25/69] listen to delete events --- app/controllers/shared/api.php | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index acbe4466e..04db524fb 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -162,7 +162,6 @@ App::init() $dbForProject->on(Database::EVENT_DOCUMENT_CREATE, function($event, Document $document) use ($usage) { $collection = $document->getCollection(); - Console::info('collection: ' . $collection); switch($collection) { case 'users': @@ -193,6 +192,38 @@ App::init() } }); + $dbForProject->on(Database::EVENT_DOCUMENT_DELETE, function($event, Document $document) use ($usage) { + $collection = $document->getCollection(); + switch($collection) + { + case 'users': + $usage->setParam('users.{scope}.count.total', -1); + break; + case 'databases': + $usage->setParam('databases.{scope}.count.total', -1); + break; + case 'buckets': + $usage->setParam('buckets.{scope}.count.total', -1); + break; + default: + if(strpos($collection, 'buckets_') === 0) { + $usage + ->setParam('bucketId', $document->getAttribute('bucketId')) + ->setParam('files.{scope}.count.total', -1); + } else if (strpos($collection, 'database_') === 0) { + if(strpos($collection, '_collection_') != false) { + $usage + ->setParam('databaseId', $document->getAttribute('databaseId')) + ->setParam('collectionId', $document->getAttribute('collectionId')) + ->setParam('files.{scope}.count.total', -1); + } else { + Console::info('Collection created'); + } + } + break; + } + }); + $useCache = $route->getLabel('cache', false); if ($useCache) { From 00d194d5d181836feafcce05e6272d795e568bec Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 17 Oct 2022 07:37:30 +0000 Subject: [PATCH 26/69] fix linters --- app/controllers/shared/api.php | 26 +++++++++---------- src/Appwrite/Usage/Calculators/TimeSeries.php | 8 +++--- 2 files changed, 16 insertions(+), 18 deletions(-) diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 04db524fb..c733b917e 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -160,26 +160,25 @@ App::init() $deletes->setProject($project); $database->setProject($project); - $dbForProject->on(Database::EVENT_DOCUMENT_CREATE, function($event, Document $document) use ($usage) { + $dbForProject->on(Database::EVENT_DOCUMENT_CREATE, function ($event, Document $document) use ($usage) { $collection = $document->getCollection(); - switch($collection) - { + switch ($collection) { case 'users': $usage->setParam('users.{scope}.count.total', 1); break; case 'databases': $usage->setParam('databases.{scope}.count.total', 1); break; - case 'buckets': + case 'buckets': $usage->setParam('buckets.{scope}.count.total', 1); break; default: - if(strpos($collection, 'buckets_') === 0) { + if (strpos($collection, 'buckets_') === 0) { $usage ->setParam('bucketId', $document->getAttribute('bucketId')) ->setParam('files.{scope}.count.total', 1); - } else if (strpos($collection, 'database_') === 0) { - if(strpos($collection, '_collection_') != false) { + } elseif (strpos($collection, 'database_') === 0) { + if (strpos($collection, '_collection_') != false) { $usage ->setParam('databaseId', $document->getAttribute('databaseId')) ->setParam('collectionId', $document->getAttribute('collectionId')) @@ -192,26 +191,25 @@ App::init() } }); - $dbForProject->on(Database::EVENT_DOCUMENT_DELETE, function($event, Document $document) use ($usage) { + $dbForProject->on(Database::EVENT_DOCUMENT_DELETE, function ($event, Document $document) use ($usage) { $collection = $document->getCollection(); - switch($collection) - { + switch ($collection) { case 'users': $usage->setParam('users.{scope}.count.total', -1); break; case 'databases': $usage->setParam('databases.{scope}.count.total', -1); break; - case 'buckets': + case 'buckets': $usage->setParam('buckets.{scope}.count.total', -1); break; default: - if(strpos($collection, 'buckets_') === 0) { + if (strpos($collection, 'buckets_') === 0) { $usage ->setParam('bucketId', $document->getAttribute('bucketId')) ->setParam('files.{scope}.count.total', -1); - } else if (strpos($collection, 'database_') === 0) { - if(strpos($collection, '_collection_') != false) { + } elseif (strpos($collection, 'database_') === 0) { + if (strpos($collection, '_collection_') != false) { $usage ->setParam('databaseId', $document->getAttribute('databaseId')) ->setParam('collectionId', $document->getAttribute('collectionId')) diff --git a/src/Appwrite/Usage/Calculators/TimeSeries.php b/src/Appwrite/Usage/Calculators/TimeSeries.php index d03a831e3..7ec6d6913 100644 --- a/src/Appwrite/Usage/Calculators/TimeSeries.php +++ b/src/Appwrite/Usage/Calculators/TimeSeries.php @@ -233,7 +233,7 @@ class TimeSeries extends Calculator 'table' => 'appwrite_usage_builds_{scope}_compute', 'groupBy' => ['functionId'], ], - + 'executions.functionId.compute.failure' => [ 'table' => 'appwrite_usage_executions_{scope}_compute', 'groupBy' => ['functionId'], @@ -312,14 +312,14 @@ class TimeSeries extends Calculator 'table' => 'appwrite_usage_files_{scope}_storage_size', 'groupBy' => ['bucketId'] ], - + 'builds.$all.compute.time' => [ 'table' => 'appwrite_usage_executions_{scope}_compute_time', ], 'executions.$all.compute.time' => [ 'table' => 'appwrite_usage_executions_{scope}_compute_time', ], - + 'executions.functionId.compute.time' => [ 'table' => 'appwrite_usage_executions_{scope}_compute_time', 'groupBy' => ['functionId'], @@ -328,7 +328,7 @@ class TimeSeries extends Calculator 'table' => 'appwrite_usage_builds_{scope}_compute_time', 'groupBy' => ['functionId'], ], - + 'project.$all.compute.time' => [ // Built time + execution time 'table' => 'appwrite_usage_project_{scope}_compute_time', 'groupBy' => ['functionId'], From 5b90c433ab880d04c57698fd8a76f9bce873b580 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 17 Oct 2022 07:53:55 +0000 Subject: [PATCH 27/69] fix collection id --- app/controllers/shared/api.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index c733b917e..418b22eb1 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -181,7 +181,7 @@ App::init() if (strpos($collection, '_collection_') != false) { $usage ->setParam('databaseId', $document->getAttribute('databaseId')) - ->setParam('collectionId', $document->getAttribute('collectionId')) + ->setParam('collectionId', $document->getAttribute('$collectionId')) ->setParam('files.{scope}.count.total', 1); } else { Console::info('Collection created'); From f71eb5a169c3b1a25ef77d7409ce8dcc5b58e063 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 17 Oct 2022 08:04:12 +0000 Subject: [PATCH 28/69] database listeners --- composer.json | 4 ++-- composer.lock | 60 ++++++++++++++++++++++++--------------------------- 2 files changed, 30 insertions(+), 34 deletions(-) diff --git a/composer.json b/composer.json index 47317ed5e..a7ebf3fc6 100644 --- a/composer.json +++ b/composer.json @@ -48,10 +48,10 @@ "utopia-php/abuse": "0.14.*", "utopia-php/analytics": "0.2.*", "utopia-php/audit": "0.15.*", - "utopia-php/cache": "0.6.*", + "utopia-php/cache": "0.7.*", "utopia-php/cli": "0.13.*", "utopia-php/config": "0.2.*", - "utopia-php/database": "0.26.*", + "utopia-php/database": "dev-main as 0.26.0", "utopia-php/locale": "0.4.*", "utopia-php/registry": "0.5.*", "utopia-php/preloader": "0.2.*", diff --git a/composer.lock b/composer.lock index 0f9c73b57..80102883a 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "08fdd139ad1285b02c4b4e555679e7de", + "content-hash": "80c8717dd836655d2a135d306ad04de1", "packages": [ { "name": "adhocore/jwt", @@ -1897,24 +1897,26 @@ }, { "name": "utopia-php/cache", - "version": "0.6.1", + "version": "0.7.0", "source": { "type": "git", "url": "https://github.com/utopia-php/cache.git", - "reference": "9889235a6d3da6cbb1f435201529da4d27c30e79" + "reference": "cd53431242c88299daea2589e21322abe97682cc" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/cache/zipball/9889235a6d3da6cbb1f435201529da4d27c30e79", - "reference": "9889235a6d3da6cbb1f435201529da4d27c30e79", + "url": "https://api.github.com/repos/utopia-php/cache/zipball/cd53431242c88299daea2589e21322abe97682cc", + "reference": "cd53431242c88299daea2589e21322abe97682cc", "shasum": "" }, "require": { "ext-json": "*", + "ext-memcached": "*", "ext-redis": "*", "php": ">=8.0" }, "require-dev": { + "laravel/pint": "1.2.*", "phpunit/phpunit": "^9.3", "vimeo/psalm": "4.13.1" }, @@ -1928,12 +1930,6 @@ "license": [ "MIT" ], - "authors": [ - { - "name": "Eldad Fux", - "email": "eldad@appwrite.io" - } - ], "description": "A simple cache library to manage application cache storing, loading and purging", "keywords": [ "cache", @@ -1944,9 +1940,9 @@ ], "support": { "issues": "https://github.com/utopia-php/cache/issues", - "source": "https://github.com/utopia-php/cache/tree/0.6.1" + "source": "https://github.com/utopia-php/cache/tree/0.7.0" }, - "time": "2022-08-10T08:12:46+00:00" + "time": "2022-10-16T06:04:12+00:00" }, { "name": "utopia-php/cli", @@ -2054,16 +2050,16 @@ }, { "name": "utopia-php/database", - "version": "0.26.0", + "version": "dev-main", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "d172af2541137c83a86d066f82f48914b5a3a610" + "reference": "4553c3d150b631f5b79a876343a71f5966681970" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/d172af2541137c83a86d066f82f48914b5a3a610", - "reference": "d172af2541137c83a86d066f82f48914b5a3a610", + "url": "https://api.github.com/repos/utopia-php/database/zipball/4553c3d150b631f5b79a876343a71f5966681970", + "reference": "4553c3d150b631f5b79a876343a71f5966681970", "shasum": "" }, "require": { @@ -2072,7 +2068,7 @@ "ext-redis": "*", "mongodb/mongodb": "1.8.0", "php": ">=8.0", - "utopia-php/cache": "0.6.*", + "utopia-php/cache": "0.7.*", "utopia-php/framework": "0.*.*" }, "require-dev": { @@ -2082,6 +2078,7 @@ "utopia-php/cli": "^0.11.0", "vimeo/psalm": "4.0.1" }, + "default-branch": true, "type": "library", "autoload": { "psr-4": { @@ -2092,16 +2089,6 @@ "license": [ "MIT" ], - "authors": [ - { - "name": "Eldad Fux", - "email": "eldad@appwrite.io" - }, - { - "name": "Brandon Leckemby", - "email": "brandon@appwrite.io" - } - ], "description": "A simple library to manage application persistency using multiple database adapters", "keywords": [ "database", @@ -2112,9 +2099,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/0.26.0" + "source": "https://github.com/utopia-php/database/tree/main" }, - "time": "2022-10-03T17:12:01+00:00" + "time": "2022-10-16T11:02:35+00:00" }, { "name": "utopia-php/domains", @@ -5357,9 +5344,18 @@ "time": "2022-09-28T08:42:51+00:00" } ], - "aliases": [], + "aliases": [ + { + "package": "utopia-php/database", + "version": "dev-main", + "alias": "0.26.0", + "alias_normalized": "0.26.0.0" + } + ], "minimum-stability": "stable", - "stability-flags": [], + "stability-flags": { + "utopia-php/database": 20 + }, "prefer-stable": false, "prefer-lowest": false, "platform": { From e96c0b11278737ef4b31735fbc78053d74b66222 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 17 Oct 2022 08:57:12 +0000 Subject: [PATCH 29/69] fix composer --- composer.json | 4 ++-- composer.lock | 43 ++++++++++++++++++++++--------------------- 2 files changed, 24 insertions(+), 23 deletions(-) diff --git a/composer.json b/composer.json index a7ebf3fc6..1e28c3965 100644 --- a/composer.json +++ b/composer.json @@ -48,10 +48,10 @@ "utopia-php/abuse": "0.14.*", "utopia-php/analytics": "0.2.*", "utopia-php/audit": "0.15.*", - "utopia-php/cache": "0.7.*", + "utopia-php/cache": "0.6.*", "utopia-php/cli": "0.13.*", "utopia-php/config": "0.2.*", - "utopia-php/database": "dev-main as 0.26.0", + "utopia-php/database": "0.27.0 as 0.26.0", "utopia-php/locale": "0.4.*", "utopia-php/registry": "0.5.*", "utopia-php/preloader": "0.2.*", diff --git a/composer.lock b/composer.lock index 80102883a..7c65a1ec6 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "80c8717dd836655d2a135d306ad04de1", + "content-hash": "0e0aa8681829f5c1c68b5b80044f12a5", "packages": [ { "name": "adhocore/jwt", @@ -1897,26 +1897,24 @@ }, { "name": "utopia-php/cache", - "version": "0.7.0", + "version": "0.6.1", "source": { "type": "git", "url": "https://github.com/utopia-php/cache.git", - "reference": "cd53431242c88299daea2589e21322abe97682cc" + "reference": "9889235a6d3da6cbb1f435201529da4d27c30e79" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/cache/zipball/cd53431242c88299daea2589e21322abe97682cc", - "reference": "cd53431242c88299daea2589e21322abe97682cc", + "url": "https://api.github.com/repos/utopia-php/cache/zipball/9889235a6d3da6cbb1f435201529da4d27c30e79", + "reference": "9889235a6d3da6cbb1f435201529da4d27c30e79", "shasum": "" }, "require": { "ext-json": "*", - "ext-memcached": "*", "ext-redis": "*", "php": ">=8.0" }, "require-dev": { - "laravel/pint": "1.2.*", "phpunit/phpunit": "^9.3", "vimeo/psalm": "4.13.1" }, @@ -1930,6 +1928,12 @@ "license": [ "MIT" ], + "authors": [ + { + "name": "Eldad Fux", + "email": "eldad@appwrite.io" + } + ], "description": "A simple cache library to manage application cache storing, loading and purging", "keywords": [ "cache", @@ -1940,9 +1944,9 @@ ], "support": { "issues": "https://github.com/utopia-php/cache/issues", - "source": "https://github.com/utopia-php/cache/tree/0.7.0" + "source": "https://github.com/utopia-php/cache/tree/0.6.1" }, - "time": "2022-10-16T06:04:12+00:00" + "time": "2022-08-10T08:12:46+00:00" }, { "name": "utopia-php/cli", @@ -2050,16 +2054,16 @@ }, { "name": "utopia-php/database", - "version": "dev-main", + "version": "0.27.0", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "4553c3d150b631f5b79a876343a71f5966681970" + "reference": "0644cd8a7efee2fc745ed70dbf2b73a78eeefa52" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/4553c3d150b631f5b79a876343a71f5966681970", - "reference": "4553c3d150b631f5b79a876343a71f5966681970", + "url": "https://api.github.com/repos/utopia-php/database/zipball/0644cd8a7efee2fc745ed70dbf2b73a78eeefa52", + "reference": "0644cd8a7efee2fc745ed70dbf2b73a78eeefa52", "shasum": "" }, "require": { @@ -2068,7 +2072,7 @@ "ext-redis": "*", "mongodb/mongodb": "1.8.0", "php": ">=8.0", - "utopia-php/cache": "0.7.*", + "utopia-php/cache": "0.6.*", "utopia-php/framework": "0.*.*" }, "require-dev": { @@ -2078,7 +2082,6 @@ "utopia-php/cli": "^0.11.0", "vimeo/psalm": "4.0.1" }, - "default-branch": true, "type": "library", "autoload": { "psr-4": { @@ -2099,9 +2102,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/main" + "source": "https://github.com/utopia-php/database/tree/0.27.0" }, - "time": "2022-10-16T11:02:35+00:00" + "time": "2022-10-16T07:39:26+00:00" }, { "name": "utopia-php/domains", @@ -5347,15 +5350,13 @@ "aliases": [ { "package": "utopia-php/database", - "version": "dev-main", + "version": "0.27.0.0", "alias": "0.26.0", "alias_normalized": "0.26.0.0" } ], "minimum-stability": "stable", - "stability-flags": { - "utopia-php/database": 20 - }, + "stability-flags": [], "prefer-stable": false, "prefer-lowest": false, "platform": { From b1f3fdce7a0b0460e773231f459fa9fd22aeb4f5 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 18 Oct 2022 02:26:28 +0000 Subject: [PATCH 30/69] fix stats test --- tests/unit/Usage/StatsTest.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/unit/Usage/StatsTest.php b/tests/unit/Usage/StatsTest.php index 0b39dfdaa..f02184139 100644 --- a/tests/unit/Usage/StatsTest.php +++ b/tests/unit/Usage/StatsTest.php @@ -38,10 +38,12 @@ class StatsTest extends TestCase { $this->object ->setParam('projectId', 'appwrite_test') + ->setParam('projectInternalId', 1) ->setParam('networkRequestSize', 100) ; $this->assertEquals('appwrite_test', $this->object->getParam('projectId')); + $this->assertEquals(1, $this->object->getParam('projectInternalId')); $this->assertEquals(100, $this->object->getParam('networkRequestSize')); $this->object->submit(); From a3f14220d346c714884edff76d60c024a14bc72d Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 18 Oct 2022 03:19:12 +0000 Subject: [PATCH 31/69] take care of negative values --- src/Appwrite/Usage/Calculators/TimeSeries.php | 2 +- src/Appwrite/Usage/Stats.php | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Appwrite/Usage/Calculators/TimeSeries.php b/src/Appwrite/Usage/Calculators/TimeSeries.php index 7ec6d6913..1a94af629 100644 --- a/src/Appwrite/Usage/Calculators/TimeSeries.php +++ b/src/Appwrite/Usage/Calculators/TimeSeries.php @@ -420,7 +420,7 @@ class TimeSeries extends Calculator $query .= "WHERE \"time\" > '{$start}' "; $query .= "AND \"time\" < '{$end}' "; $query .= "AND \"metric_type\"='counter' {$filters} "; - $query .= "GROUP BY time({$period['key']}), \"projectId\" {$groupBy} "; + $query .= "GROUP BY time({$period['key']}), \"projectId\", \"projectInternalId\" {$groupBy} "; $query .= "FILL(null)"; try { diff --git a/src/Appwrite/Usage/Stats.php b/src/Appwrite/Usage/Stats.php index efd83a2dc..d26f44c74 100644 --- a/src/Appwrite/Usage/Stats.php +++ b/src/Appwrite/Usage/Stats.php @@ -108,7 +108,7 @@ class Stats foreach ($usersMetrics as $metric) { $value = $this->params[$metric] ?? 0; - if ($value >= 1) { + if ($value == 1 || $value == -1) { $this->statsd->count($metric . $tags, $value); } } @@ -133,7 +133,7 @@ class Stats foreach ($dbMetrics as $metric) { $value = $this->params[$metric] ?? 0; - if ($value >= 1) { + if ($value == 1 || $value == -1) { $dbTags = $tags . ",collectionId=" . ($this->params['collectionId'] ?? '') . ",databaseId=" . ($this->params['databaseId'] ?? ''); $this->statsd->count($metric . $dbTags, $value); } @@ -154,7 +154,7 @@ class Stats foreach ($storageMertics as $metric) { $value = $this->params[$metric] ?? 0; - if ($value >= 1) { + if ($value == 1 || $value == -1) { $storageTags = $tags . ",bucketId=" . ($this->params['bucketId'] ?? ''); $this->statsd->count($metric . $storageTags, $value); } From 8093f034c0f13f29f11c39af94c692eec4aa9a8c Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 18 Oct 2022 03:53:07 +0000 Subject: [PATCH 32/69] fix inbound and outbount --- src/Appwrite/Usage/Stats.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Appwrite/Usage/Stats.php b/src/Appwrite/Usage/Stats.php index d26f44c74..8bb83b264 100644 --- a/src/Appwrite/Usage/Stats.php +++ b/src/Appwrite/Usage/Stats.php @@ -92,8 +92,8 @@ class Stats $this->statsd->increment('project.{scope}.network.requests' . $tags . ',method=' . \strtolower($httpMethod)); } - $inbound = $this->params['networkRequestSize'] ?? 0; - $outbound = $this->params['networkResponseSize'] ?? 0; + $inbound = $this->params['project.{scope}.network.inbound'] ?? 0; + $outbound = $this->params['project.{scope}.network.outbound'] ?? 0; $this->statsd->count('project.{scope}.network.inbound' . $tags, $inbound); $this->statsd->count('project.{scope}.network.outbound' . $tags, $outbound); $this->statsd->count('project.{scope}.network.bandwidth' . $tags, $inbound + $outbound); From d7f87051d61b133a75a42fd551a90a85c45b35b2 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 18 Oct 2022 06:10:33 +0000 Subject: [PATCH 33/69] fix test --- tests/e2e/General/UsageTest.php | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/e2e/General/UsageTest.php b/tests/e2e/General/UsageTest.php index 0c42c42ff..525867a81 100644 --- a/tests/e2e/General/UsageTest.php +++ b/tests/e2e/General/UsageTest.php @@ -114,6 +114,7 @@ class UsageTest extends Scope 'x-appwrite-project' => $projectId, 'x-appwrite-mode' => 'admin' ])); + $requestsCount++; $res = $res['body']; $this->assertEquals(10, $res['usersCreate'][array_key_last($res['usersCreate'])]['value']); $this->validateDates($res['usersCreate']); @@ -279,6 +280,7 @@ class UsageTest extends Scope 'x-appwrite-project' => $projectId, 'x-appwrite-mode' => 'admin' ])); + $requestsCount++; $res = $res['body']; $this->assertEquals($storageTotal, $res['storage'][array_key_last($res['storage'])]['value']); $this->validateDates($res['storage']); @@ -303,6 +305,7 @@ class UsageTest extends Scope 'x-appwrite-project' => $projectId, 'x-appwrite-mode' => 'admin' ])); + $requestsCount++; $res = $res['body']; $this->assertEquals($storageTotal, $res['filesStorage'][array_key_last($res['filesStorage'])]['value']); $this->assertEquals($filesCount, $res['filesCount'][array_key_last($res['filesCount'])]['value']); From 1fdaaeef71c85fd24674de781d781e61e71ebb9f Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 18 Oct 2022 06:19:57 +0000 Subject: [PATCH 34/69] reduce waiting time --- tests/e2e/General/UsageTest.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/e2e/General/UsageTest.php b/tests/e2e/General/UsageTest.php index 525867a81..197be3165 100644 --- a/tests/e2e/General/UsageTest.php +++ b/tests/e2e/General/UsageTest.php @@ -85,7 +85,7 @@ class UsageTest extends Scope #[Retry(count: 1)] public function testUsersStats(array $data): array { - sleep(35); + sleep(10); $projectId = $data['projectId']; $headers = $data['headers']; @@ -256,7 +256,7 @@ class UsageTest extends Scope $filesCreate = $data['filesCreate']; $filesDelete = $data['filesDelete']; - sleep(35); + sleep(10); // console request $headers = [ @@ -496,7 +496,7 @@ class UsageTest extends Scope $documentsRead = $data['documentsRead']; $documentsDelete = $data['documentsDelete']; - sleep(35); + sleep(10); // check datbase stats $headers = [ @@ -704,7 +704,7 @@ class UsageTest extends Scope $executions = $data['executions']; $failures = $data['failures']; - sleep(25); + sleep(10); $response = $this->client->call(Client::METHOD_GET, '/functions/' . $functionId . '/usage', $headers, [ 'range' => '30d' From 305dafe2ad6caa7560d4236e3177c853a128a958 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 18 Oct 2022 06:53:17 +0000 Subject: [PATCH 35/69] update missing stats --- app/controllers/shared/api.php | 10 +++++--- src/Appwrite/Usage/Calculators/TimeSeries.php | 14 +++++++++++ src/Appwrite/Usage/Stats.php | 24 ++++++++++++++----- 3 files changed, 39 insertions(+), 9 deletions(-) diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 418b22eb1..376864e3b 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -172,17 +172,21 @@ App::init() case 'buckets': $usage->setParam('buckets.{scope}.count.total', 1); break; + case 'deployments': + $usage->setParam('deployments.{scope}.storage.size', $document->getAttribute('size')); + break; default: if (strpos($collection, 'buckets_') === 0) { $usage ->setParam('bucketId', $document->getAttribute('bucketId')) + ->setParam('files.{scope}.storage.size', $document->getAttribute('sizeOriginal')) ->setParam('files.{scope}.count.total', 1); } elseif (strpos($collection, 'database_') === 0) { if (strpos($collection, '_collection_') != false) { $usage - ->setParam('databaseId', $document->getAttribute('databaseId')) - ->setParam('collectionId', $document->getAttribute('$collectionId')) - ->setParam('files.{scope}.count.total', 1); + ->setParam('databaseId', $document->getAttribute('databaseId')) + ->setParam('collectionId', $document->getAttribute('$collectionId')) + ->setParam('files.{scope}.count.total', 1); } else { Console::info('Collection created'); } diff --git a/src/Appwrite/Usage/Calculators/TimeSeries.php b/src/Appwrite/Usage/Calculators/TimeSeries.php index 1a94af629..edf0011b8 100644 --- a/src/Appwrite/Usage/Calculators/TimeSeries.php +++ b/src/Appwrite/Usage/Calculators/TimeSeries.php @@ -333,6 +333,20 @@ class TimeSeries extends Calculator 'table' => 'appwrite_usage_project_{scope}_compute_time', 'groupBy' => ['functionId'], ], + + 'deployments.$all.storage.size' => [ + 'table' => 'appwrite_usage_deployments_{scope}_storage_size' + ], + 'project.$all.storage.size' => [ + 'table' => 'appwrite_usage_project_{scope}_storage_size' + ], + 'files.$all.storage.size' => [ + 'table' => 'appwrite_usage_files_{scope}_storage_size' + ], + 'files.$bucketId.storage.size' => [ + 'table' => 'appwrite_usage_files_{scope}_storage_size', + 'groupBy' => ['bucketId'] + ] ]; public function __construct(Database $database, InfluxDatabase $influxDB, callable $errorHandler = null) diff --git a/src/Appwrite/Usage/Stats.php b/src/Appwrite/Usage/Stats.php index 8bb83b264..f8d16a13b 100644 --- a/src/Appwrite/Usage/Stats.php +++ b/src/Appwrite/Usage/Stats.php @@ -149,7 +149,8 @@ class Stats 'files.{scope}.requests.update', 'files.{scope}.requests.delete', 'buckets.{scope}.count.total', - 'files.{scope}.count.total' + 'files.{scope}.count.total', + 'files.{scope}.storage.size' ]; foreach ($storageMertics as $metric) { @@ -183,19 +184,30 @@ class Stats $functionBuildTime = ($this->params['buildTime'] ?? 0) * 1000; // ms $functionBuildStatus = $this->params['buildStatus'] ?? ''; $functionCompute = $functionExecutionTime + $functionBuildTime; + $functionTags = $tags . ',functionId=' . $functionId; + + $deploymentSize = $this->params['deployment.{scope}.storage.size'] ?? 0; + $storageSize = $this->params['files.{scope}.storage.size'] ?? 0; + if($deploymentSize + $storageSize) { + $this->statsd->count('project.{scope}.storage.size' . $tags, $deploymentSize + $storageSize); + } + + if($deploymentSize > 0) { + $this->statsd->count('deployments.{scope}.storage.size' . $functionTags, $deploymentSize); + } if ($functionExecution >= 1) { - $this->statsd->increment('executions.{scope}.compute' . $tags . ',functionId=' . $functionId . ',functionStatus=' . $functionExecutionStatus); + $this->statsd->increment('executions.{scope}.compute' . $functionTags . ',functionStatus=' . $functionExecutionStatus); if ($functionExecutionTime > 0) { - $this->statsd->count('executions.{scope}.compute.time' . $tags . ',functionId=' . $functionId, $functionExecutionTime); + $this->statsd->count('executions.{scope}.compute.time' . $functionTags, $functionExecutionTime); } } if ($functionBuild >= 1) { - $this->statsd->increment('builds.{scope}.compute' . $tags . ',functionId=' . $functionId . ',functionBuildStatus=' . $functionBuildStatus); - $this->statsd->count('builds.{scope}.compute.time' . $tags . ',functionId=' . $functionId, $functionBuildTime); + $this->statsd->increment('builds.{scope}.compute' . $functionTags . ',functionBuildStatus=' . $functionBuildStatus); + $this->statsd->count('builds.{scope}.compute.time' . $functionTags, $functionBuildTime); } if ($functionBuild + $functionExecution >= 1) { - $this->statsd->count('project.{scope}.compute.time' . $tags . ',functionId=' . $functionId, $functionCompute); + $this->statsd->count('project.{scope}.compute.time' . $functionTags, $functionCompute); } $this->reset(); From e7c82a28dd04dc3e5159a404308d5409276c79cc Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 18 Oct 2022 07:09:11 +0000 Subject: [PATCH 36/69] more fixes --- app/controllers/shared/api.php | 4 ++++ src/Appwrite/Usage/Stats.php | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 376864e3b..690287f27 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -207,10 +207,14 @@ App::init() case 'buckets': $usage->setParam('buckets.{scope}.count.total', -1); break; + case 'deployments': + $usage->setParam('deployments.{scope}.storage.size', -$document->getAttribute('size')); + break; default: if (strpos($collection, 'buckets_') === 0) { $usage ->setParam('bucketId', $document->getAttribute('bucketId')) + ->setParam('files.{scope}.storage.size', -$document->getAttribute('sizeOriginal')) ->setParam('files.{scope}.count.total', -1); } elseif (strpos($collection, 'database_') === 0) { if (strpos($collection, '_collection_') != false) { diff --git a/src/Appwrite/Usage/Stats.php b/src/Appwrite/Usage/Stats.php index f8d16a13b..3608b9491 100644 --- a/src/Appwrite/Usage/Stats.php +++ b/src/Appwrite/Usage/Stats.php @@ -188,7 +188,7 @@ class Stats $deploymentSize = $this->params['deployment.{scope}.storage.size'] ?? 0; $storageSize = $this->params['files.{scope}.storage.size'] ?? 0; - if($deploymentSize + $storageSize) { + if($deploymentSize + $storageSize > 0) { $this->statsd->count('project.{scope}.storage.size' . $tags, $deploymentSize + $storageSize); } From 027dafe90625ae73794b209d03734cc81133dd91 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 18 Oct 2022 07:26:28 +0000 Subject: [PATCH 37/69] fix linters --- src/Appwrite/Usage/Stats.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Appwrite/Usage/Stats.php b/src/Appwrite/Usage/Stats.php index 3608b9491..20007579e 100644 --- a/src/Appwrite/Usage/Stats.php +++ b/src/Appwrite/Usage/Stats.php @@ -185,14 +185,14 @@ class Stats $functionBuildStatus = $this->params['buildStatus'] ?? ''; $functionCompute = $functionExecutionTime + $functionBuildTime; $functionTags = $tags . ',functionId=' . $functionId; - + $deploymentSize = $this->params['deployment.{scope}.storage.size'] ?? 0; $storageSize = $this->params['files.{scope}.storage.size'] ?? 0; - if($deploymentSize + $storageSize > 0) { + if ($deploymentSize + $storageSize > 0) { $this->statsd->count('project.{scope}.storage.size' . $tags, $deploymentSize + $storageSize); } - if($deploymentSize > 0) { + if ($deploymentSize > 0) { $this->statsd->count('deployments.{scope}.storage.size' . $functionTags, $deploymentSize); } From 8cd406033416c54ec8a16eaca7fdff1c71bc1603 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 18 Oct 2022 07:38:05 +0000 Subject: [PATCH 38/69] fix storage stats --- src/Appwrite/Usage/Stats.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Usage/Stats.php b/src/Appwrite/Usage/Stats.php index 20007579e..06d43ea0c 100644 --- a/src/Appwrite/Usage/Stats.php +++ b/src/Appwrite/Usage/Stats.php @@ -155,7 +155,7 @@ class Stats foreach ($storageMertics as $metric) { $value = $this->params[$metric] ?? 0; - if ($value == 1 || $value == -1) { + if ($value >= 1 || $value == -1) { $storageTags = $tags . ",bucketId=" . ($this->params['bucketId'] ?? ''); $this->statsd->count($metric . $storageTags, $value); } From 868e487a5909b6b13771166565a926d4b5b2bb91 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 18 Oct 2022 07:52:41 +0000 Subject: [PATCH 39/69] fix bucket collection name --- app/controllers/shared/api.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 690287f27..a36668b95 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -176,7 +176,7 @@ App::init() $usage->setParam('deployments.{scope}.storage.size', $document->getAttribute('size')); break; default: - if (strpos($collection, 'buckets_') === 0) { + if (strpos($collection, 'bucket_') === 0) { $usage ->setParam('bucketId', $document->getAttribute('bucketId')) ->setParam('files.{scope}.storage.size', $document->getAttribute('sizeOriginal')) @@ -211,7 +211,7 @@ App::init() $usage->setParam('deployments.{scope}.storage.size', -$document->getAttribute('size')); break; default: - if (strpos($collection, 'buckets_') === 0) { + if (strpos($collection, 'bucket_') === 0) { $usage ->setParam('bucketId', $document->getAttribute('bucketId')) ->setParam('files.{scope}.storage.size', -$document->getAttribute('sizeOriginal')) From 5ff8ac46aa794417c48272d692da2a409133cb20 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 18 Oct 2022 08:28:12 +0000 Subject: [PATCH 40/69] fix negative stats --- src/Appwrite/Usage/Stats.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Appwrite/Usage/Stats.php b/src/Appwrite/Usage/Stats.php index 06d43ea0c..ba68d9ddf 100644 --- a/src/Appwrite/Usage/Stats.php +++ b/src/Appwrite/Usage/Stats.php @@ -155,7 +155,7 @@ class Stats foreach ($storageMertics as $metric) { $value = $this->params[$metric] ?? 0; - if ($value >= 1 || $value == -1) { + if ($value >= 1 || $value == -1 || $value < -1) { $storageTags = $tags . ",bucketId=" . ($this->params['bucketId'] ?? ''); $this->statsd->count($metric . $storageTags, $value); } @@ -188,11 +188,11 @@ class Stats $deploymentSize = $this->params['deployment.{scope}.storage.size'] ?? 0; $storageSize = $this->params['files.{scope}.storage.size'] ?? 0; - if ($deploymentSize + $storageSize > 0) { + if ($deploymentSize + $storageSize > 0 || $deploymentSize + $storageSize <= -1) { $this->statsd->count('project.{scope}.storage.size' . $tags, $deploymentSize + $storageSize); } - if ($deploymentSize > 0) { + if ($deploymentSize > 0 || $deploymentSize <= -1) { $this->statsd->count('deployments.{scope}.storage.size' . $functionTags, $deploymentSize); } From d1b3ca517878eb9d1465a7db09bd6c73a23f4d83 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Fri, 21 Oct 2022 01:29:13 +0000 Subject: [PATCH 41/69] comment database worker --- docker-compose.yml | 71 ++++++++++++++++++++++++---------------------- 1 file changed, 37 insertions(+), 34 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index e69c14282..5e041409b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -602,40 +602,43 @@ services: - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG - appwrite-usage-database: - entrypoint: - - usage - - --type=database - <<: *x-logging - container_name: appwrite-usage-database - image: appwrite-dev - networks: - - appwrite - volumes: - - ./app:/usr/src/code/app - - ./src:/usr/src/code/src - - ./dev:/usr/local/dev - depends_on: - - influxdb - - mariadb - environment: - - _APP_ENV - - _APP_OPENSSL_KEY_V1 - - _APP_DB_HOST - - _APP_DB_PORT - - _APP_DB_SCHEMA - - _APP_DB_USER - - _APP_DB_PASS - - _APP_INFLUXDB_HOST - - _APP_INFLUXDB_PORT - - _APP_USAGE_TIMESERIES_INTERVAL - - _APP_USAGE_DATABASE_INTERVAL - - _APP_REDIS_HOST - - _APP_REDIS_PORT - - _APP_REDIS_USER - - _APP_REDIS_PASS - - _APP_LOGGING_PROVIDER - - _APP_LOGGING_CONFIG + # appwrite-usage-database: + # entrypoint: + # - usage + # - --type=database + # <<: *x-logging + # container_name: appwrite-usage-database + # build: + # context: . + # args: + # - DEBUG=false + # networks: + # - appwrite + # volumes: + # - ./app:/usr/src/code/app + # - ./src:/usr/src/code/src + # - ./dev:/usr/local/dev + # depends_on: + # - influxdb + # - mariadb + # environment: + # - _APP_ENV + # - _APP_OPENSSL_KEY_V1 + # - _APP_DB_HOST + # - _APP_DB_PORT + # - _APP_DB_SCHEMA + # - _APP_DB_USER + # - _APP_DB_PASS + # - _APP_INFLUXDB_HOST + # - _APP_INFLUXDB_PORT + # - _APP_USAGE_TIMESERIES_INTERVAL + # - _APP_USAGE_DATABASE_INTERVAL + # - _APP_REDIS_HOST + # - _APP_REDIS_PORT + # - _APP_REDIS_USER + # - _APP_REDIS_PASS + # - _APP_LOGGING_PROVIDER + # - _APP_LOGGING_CONFIG appwrite-schedule: entrypoint: schedule From 088fd892943606e2b085f5c98104e917a4b545c6 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Fri, 21 Oct 2022 01:40:26 +0000 Subject: [PATCH 42/69] fix metric name --- src/Appwrite/Usage/Calculators/TimeSeries.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Usage/Calculators/TimeSeries.php b/src/Appwrite/Usage/Calculators/TimeSeries.php index edf0011b8..ca78769f9 100644 --- a/src/Appwrite/Usage/Calculators/TimeSeries.php +++ b/src/Appwrite/Usage/Calculators/TimeSeries.php @@ -343,7 +343,7 @@ class TimeSeries extends Calculator 'files.$all.storage.size' => [ 'table' => 'appwrite_usage_files_{scope}_storage_size' ], - 'files.$bucketId.storage.size' => [ + 'files.bucketId.storage.size' => [ 'table' => 'appwrite_usage_files_{scope}_storage_size', 'groupBy' => ['bucketId'] ] From a7e1e5f25f44f736bc54515e172024ca11a92968 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Fri, 21 Oct 2022 01:40:33 +0000 Subject: [PATCH 43/69] fix period --- app/controllers/api/storage.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index f23628574..781825704 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -1571,7 +1571,7 @@ App::get('/v1/storage/:bucketId/usage') if (App::getEnv('_APP_USAGE_STATS', 'enabled') === 'enabled') { $periods = [ '24h' => [ - 'period' => '30m', + 'period' => '1h', 'limit' => 48, ], '7d' => [ From d35b438255f647d831802dfa7847d8b9679ce077 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Fri, 21 Oct 2022 01:50:03 +0000 Subject: [PATCH 44/69] fix collection and document counts --- app/controllers/shared/api.php | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index a36668b95..c6597c339 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -182,13 +182,14 @@ App::init() ->setParam('files.{scope}.storage.size', $document->getAttribute('sizeOriginal')) ->setParam('files.{scope}.count.total', 1); } elseif (strpos($collection, 'database_') === 0) { + $usage + ->setParam('databaseId', $document->getAttribute('databaseId')); if (strpos($collection, '_collection_') != false) { $usage - ->setParam('databaseId', $document->getAttribute('databaseId')) ->setParam('collectionId', $document->getAttribute('$collectionId')) - ->setParam('files.{scope}.count.total', 1); + ->setParam('documents.{scope}.count.total', 1); } else { - Console::info('Collection created'); + $usage->setParam('collections.{scope}.count.total', 1); } } break; @@ -217,13 +218,14 @@ App::init() ->setParam('files.{scope}.storage.size', -$document->getAttribute('sizeOriginal')) ->setParam('files.{scope}.count.total', -1); } elseif (strpos($collection, 'database_') === 0) { + $usage + ->setParam('databaseId', $document->getAttribute('databaseId')); if (strpos($collection, '_collection_') != false) { $usage - ->setParam('databaseId', $document->getAttribute('databaseId')) - ->setParam('collectionId', $document->getAttribute('collectionId')) - ->setParam('files.{scope}.count.total', -1); + ->setParam('collectionId', $document->getAttribute('collectionId')) + ->setParam('documents.{scope}.count.total', -1); } else { - Console::info('Collection created'); + $usage->setParam('collections.{scope}.count.total', -1); } } break; From 9195c813dbecb1654c912a5787bf72821741ddec Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Fri, 21 Oct 2022 01:53:39 +0000 Subject: [PATCH 45/69] missing documents total count --- src/Appwrite/Usage/Calculators/TimeSeries.php | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Appwrite/Usage/Calculators/TimeSeries.php b/src/Appwrite/Usage/Calculators/TimeSeries.php index ca78769f9..c58722838 100644 --- a/src/Appwrite/Usage/Calculators/TimeSeries.php +++ b/src/Appwrite/Usage/Calculators/TimeSeries.php @@ -283,9 +283,8 @@ class TimeSeries extends Calculator 'collections.$all.count.total' => [ 'table' => 'appwrite_usage_collections_{scope}_count_total', ], - 'documents.databaseId.count.total' => [ + 'documents.$all.count.total' => [ 'table' => 'appwrite_usage_documents_{scope}_count_total', - 'groupBy' => ['databaseId'] ], 'collections.databaseId.count.total' => [ 'table' => 'appwrite_usage_collections_{scope}_count_total', From dfa35e0a0b86dac40b45cfded7a4c5a0ffc11179 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Fri, 21 Oct 2022 05:45:25 +0000 Subject: [PATCH 46/69] fix periods on usage endpoints --- app/controllers/api/functions.php | 12 ++++++------ app/controllers/api/projects.php | 6 +++--- app/controllers/api/storage.php | 10 +++++----- app/controllers/api/users.php | 6 +++--- 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index d3a26b414..a8df29582 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -237,8 +237,8 @@ App::get('/v1/functions/:functionId/usage') if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { $periods = [ '24h' => [ - 'period' => '30m', - 'limit' => 48, + 'period' => '1h', + 'limit' => 24, ], '7d' => [ 'period' => '1d', @@ -292,7 +292,7 @@ App::get('/v1/functions/:functionId/usage') while ($backfill > 0) { $last = $limit - $backfill - 1; // array index of last added metric $diff = match ($period) { // convert period to seconds for unix timestamp math - '30m' => 1800, + '1h' => 3600, '1d' => 86400, }; $stats[$metric][] = [ @@ -340,8 +340,8 @@ App::get('/v1/functions/usage') if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { $periods = [ '24h' => [ - 'period' => '30m', - 'limit' => 48, + 'period' => '1h', + 'limit' => 24, ], '7d' => [ 'period' => '1d', @@ -395,7 +395,7 @@ App::get('/v1/functions/usage') while ($backfill > 0) { $last = $limit - $backfill - 1; // array index of last added metric $diff = match ($period) { // convert period to seconds for unix timestamp math - '30m' => 1800, + '1h' => 3600, '1d' => 86400, }; $stats[$metric][] = [ diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index bfea863d2..1ed6bd62d 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -269,8 +269,8 @@ App::get('/v1/projects/:projectId/usage') if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { $periods = [ '24h' => [ - 'period' => '30m', - 'limit' => 48, + 'period' => '1h', + 'limit' => 24, ], '7d' => [ 'period' => '1d', @@ -325,7 +325,7 @@ App::get('/v1/projects/:projectId/usage') while ($backfill > 0) { $last = $limit - $backfill - 1; // array index of last added metric $diff = match ($period) { // convert period to seconds for unix timestamp math - '30m' => 1800, + '1h' => 3600, '1d' => 86400, }; $stats[$metric][] = [ diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index 781825704..176d32081 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -1454,8 +1454,8 @@ App::get('/v1/storage/usage') if (App::getEnv('_APP_USAGE_STATS', 'enabled') === 'enabled') { $periods = [ '24h' => [ - 'period' => '30m', - 'limit' => 48, + 'period' => '1h', + 'limit' => 24, ], '7d' => [ 'period' => '1d', @@ -1513,7 +1513,7 @@ App::get('/v1/storage/usage') while ($backfill > 0) { $last = $limit - $backfill - 1; // array index of last added metric $diff = match ($period) { // convert period to seconds for unix timestamp math - '30m' => 1800, + '1h' => 3600, '1d' => 86400, }; $stats[$metric][] = [ @@ -1572,7 +1572,7 @@ App::get('/v1/storage/:bucketId/usage') $periods = [ '24h' => [ 'period' => '1h', - 'limit' => 48, + 'limit' => 24, ], '7d' => [ 'period' => '1d', @@ -1624,7 +1624,7 @@ App::get('/v1/storage/:bucketId/usage') while ($backfill > 0) { $last = $limit - $backfill - 1; // array index of last added metric $diff = match ($period) { // convert period to seconds for unix timestamp math - '30m' => 1800, + '1h' => 3600, '1d' => 86400, }; $stats[$metric][] = [ diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index c95105b77..6211176a0 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -1116,8 +1116,8 @@ App::get('/v1/users/usage') if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { $periods = [ '24h' => [ - 'period' => '30m', - 'limit' => 48, + 'period' => '1h', + 'limit' => 24, ], '7d' => [ 'period' => '1d', @@ -1171,7 +1171,7 @@ App::get('/v1/users/usage') while ($backfill > 0) { $last = $limit - $backfill - 1; // array index of last added metric $diff = match ($period) { // convert period to seconds for unix timestamp math - '30m' => 1800, + '1h' => 3600, '1d' => 86400, }; $stats[$metric][] = [ From 5fe2102e2684afeff9e0ae0105f695b3559b2c9f Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Fri, 21 Oct 2022 07:25:42 +0000 Subject: [PATCH 47/69] remove redundant project --- tests/e2e/General/HTTPTest.php | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/e2e/General/HTTPTest.php b/tests/e2e/General/HTTPTest.php index 06ceada97..fd5ac43ea 100644 --- a/tests/e2e/General/HTTPTest.php +++ b/tests/e2e/General/HTTPTest.php @@ -154,10 +154,6 @@ class HTTPTest extends Scope public function testSpecOpenAPI3() { - $response = $this->client->call(Client::METHOD_GET, '/specs/open-api3?platform=console', [ - 'content-type' => 'application/json', - ], []); - $directory = __DIR__ . '/../../../app/config/specs/'; $files = scandir($directory); From 3ed3ef9d716c820530dbe69015acf1b8eff7f121 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 23 Oct 2022 04:46:23 +0000 Subject: [PATCH 48/69] rename variable --- app/tasks/maintenance.php | 10 +++++----- app/workers/deletes.php | 12 ++++++------ src/Appwrite/Event/Delete.php | 10 +++++----- src/Appwrite/Usage/Calculators/Aggregator.php | 2 +- src/Appwrite/Usage/Calculators/Database.php | 6 +++--- 5 files changed, 20 insertions(+), 20 deletions(-) diff --git a/app/tasks/maintenance.php b/app/tasks/maintenance.php index 42b5ed00d..e19870289 100644 --- a/app/tasks/maintenance.php +++ b/app/tasks/maintenance.php @@ -78,12 +78,12 @@ $cli ->trigger(); } - function notifyDeleteUsageStats(int $interval30m, int $interval1d) + function notifyDeleteUsageStats(int $interval1h, int $interval1d) { (new Delete()) ->setType(DELETE_TYPE_USAGE) ->setDateTime1d(DateTime::addSeconds(new \DateTime(), -1 * $interval1d)) - ->setDateTime30m(DateTime::addSeconds(new \DateTime(), -1 * $interval30m)) + ->setDateTime1h(DateTime::addSeconds(new \DateTime(), -1 * $interval1h)) ->trigger(); } @@ -144,11 +144,11 @@ $cli $executionLogsRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_EXECUTION', '1209600'); $auditLogRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT', '1209600'); $abuseLogsRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_ABUSE', '86400'); - $usageStatsRetention30m = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_USAGE_30M', '129600'); //36 hours + $usageStatsRetention1h = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_USAGE_30M', '129600'); //36 hours $usageStatsRetention1d = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_USAGE_1D', '8640000'); // 100 days $cacheRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_CACHE', '2592000'); // 30 days - Console::loop(function () use ($interval, $executionLogsRetention, $abuseLogsRetention, $auditLogRetention, $usageStatsRetention30m, $usageStatsRetention1d, $cacheRetention) { + Console::loop(function () use ($interval, $executionLogsRetention, $abuseLogsRetention, $auditLogRetention, $usageStatsRetention1h, $usageStatsRetention1d, $cacheRetention) { $database = getConsoleDB(); $time = DateTime::now(); @@ -157,7 +157,7 @@ $cli notifyDeleteExecutionLogs($executionLogsRetention); notifyDeleteAbuseLogs($abuseLogsRetention); notifyDeleteAuditLogs($auditLogRetention); - notifyDeleteUsageStats($usageStatsRetention30m, $usageStatsRetention1d); + notifyDeleteUsageStats($usageStatsRetention1h, $usageStatsRetention1d); notifyDeleteConnections(); notifyDeleteExpiredSessions(); renewCertificates($database); diff --git a/app/workers/deletes.php b/app/workers/deletes.php index b015043b1..a5db6db5f 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -105,7 +105,7 @@ class DeletesV1 extends Worker break; case DELETE_TYPE_USAGE: - $this->deleteUsageStats($this->args['dateTime1d'], $this->args['dateTime30m']); + $this->deleteUsageStats($this->args['dateTime1d'], $this->args['dateTime1h']); break; case DELETE_TYPE_CACHE_BY_RESOURCE: @@ -215,11 +215,11 @@ class DeletesV1 extends Worker /** * @param string $datetime1d - * @param string $datetime30m + * @param string $datetime1h */ - protected function deleteUsageStats(string $datetime1d, string $datetime30m) + protected function deleteUsageStats(string $datetime1d, string $datetime1h) { - $this->deleteForProjectIds(function (string $projectId) use ($datetime1d, $datetime30m) { + $this->deleteForProjectIds(function (string $projectId) use ($datetime1d, $datetime1h) { $dbForProject = $this->getProjectDB($projectId); // Delete Usage stats $this->deleteByGroup('stats', [ @@ -228,8 +228,8 @@ class DeletesV1 extends Worker ], $dbForProject); $this->deleteByGroup('stats', [ - Query::lessThan('time', $datetime30m), - Query::equal('period', ['30m']), + Query::lessThan('time', $datetime1h), + Query::equal('period', ['1h']), ], $dbForProject); }); } diff --git a/src/Appwrite/Event/Delete.php b/src/Appwrite/Event/Delete.php index 72ace2a86..cd2461c6e 100644 --- a/src/Appwrite/Event/Delete.php +++ b/src/Appwrite/Event/Delete.php @@ -11,7 +11,7 @@ class Delete extends Event protected ?Document $document = null; protected ?string $resource = null; protected ?string $datetime = null; - protected ?string $dateTime30m = null; + protected ?string $dateTime1h = null; protected ?string $dateTime1d = null; @@ -68,14 +68,14 @@ class Delete extends Event } /** - * Sets datetime for 30m interval. + * Sets datetime for 1h interval. * * @param string $datetime * @return self */ - public function setDateTime30m(string $datetime): self + public function setDateTime1h(string $datetime): self { - $this->dateTime30m = $datetime; + $this->dateTime1h = $datetime; return $this; } @@ -141,7 +141,7 @@ class Delete extends Event 'resource' => $this->resource, 'datetime' => $this->datetime, 'dateTime1d' => $this->dateTime1d, - 'dateTime30m' => $this->dateTime30m, + 'dateTime1h' => $this->dateTime1h, ]); } } diff --git a/src/Appwrite/Usage/Calculators/Aggregator.php b/src/Appwrite/Usage/Calculators/Aggregator.php index 67cb18fe5..f28557a06 100644 --- a/src/Appwrite/Usage/Calculators/Aggregator.php +++ b/src/Appwrite/Usage/Calculators/Aggregator.php @@ -187,7 +187,7 @@ class Aggregator extends Database $this->database->setNamespace('_' . $projectId); $value = (int) $this->database->sum('stats', 'value', [ Query::equal('metric', [$metric]), - Query::equal('period', ['30m']), + Query::equal('period', ['1h']), Query::greaterThanEqual('time', $beginOfDay), Query::lessThanEqual('time', $endOfDay), ]); diff --git a/src/Appwrite/Usage/Calculators/Database.php b/src/Appwrite/Usage/Calculators/Database.php index 74179fab0..d760e6a0c 100644 --- a/src/Appwrite/Usage/Calculators/Database.php +++ b/src/Appwrite/Usage/Calculators/Database.php @@ -15,8 +15,8 @@ class Database extends Calculator { protected array $periods = [ [ - 'key' => '30m', - 'multiplier' => 1800, + 'key' => '1h', + 'multiplier' => 3600, ], [ 'key' => '1d', @@ -48,7 +48,7 @@ class Database extends Calculator foreach ($this->periods as $options) { $period = $options['key']; $date = new \DateTime(); - if ($period === '30m') { + if ($period === '1h') { $minutes = $date->format('i') >= '30' ? "30" : "00"; $time = $date->format('Y-m-d H:' . $minutes . ':00'); } elseif ($period === '1d') { From 8048d2485769e6fdfbd8c6afff97f9cebe8b67a1 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 23 Oct 2022 05:17:50 +0000 Subject: [PATCH 49/69] deprecate database aggregation --- .env | 3 +- Dockerfile | 3 +- app/config/variables.php | 4 +- app/tasks/usage.php | 62 +-- app/views/install/compose.phtml | 43 +-- docker-compose.yml | 49 +-- src/Appwrite/Usage/Calculators/Aggregator.php | 231 ----------- src/Appwrite/Usage/Calculators/Database.php | 360 ------------------ 8 files changed, 26 insertions(+), 729 deletions(-) delete mode 100644 src/Appwrite/Usage/Calculators/Aggregator.php delete mode 100644 src/Appwrite/Usage/Calculators/Database.php diff --git a/.env b/.env index 65fb54cb0..ea5738cc9 100644 --- a/.env +++ b/.env @@ -76,8 +76,7 @@ _APP_MAINTENANCE_RETENTION_CACHE=2592000 _APP_MAINTENANCE_RETENTION_EXECUTION=1209600 _APP_MAINTENANCE_RETENTION_ABUSE=86400 _APP_MAINTENANCE_RETENTION_AUDIT=1209600 -_APP_USAGE_TIMESERIES_INTERVAL=2 -_APP_USAGE_DATABASE_INTERVAL=15 +_APP_USAGE_AGGREGATION_INTERVAL=5 _APP_USAGE_STATS=enabled _APP_LOGGING_PROVIDER= _APP_LOGGING_CONFIG= diff --git a/Dockerfile b/Dockerfile index a7cae3850..2635b97e0 100755 --- a/Dockerfile +++ b/Dockerfile @@ -246,8 +246,7 @@ ENV _APP_SERVER=swoole \ _APP_SETUP=self-hosted \ _APP_VERSION=$VERSION \ _APP_USAGE_STATS=enabled \ - _APP_USAGE_TIMESERIES_INTERVAL=30 \ - _APP_USAGE_DATABASE_INTERVAL=900 \ + _APP_USAGE_AGGREGATION_INTERVAL=30 \ # 14 Days = 1209600 s _APP_MAINTENANCE_RETENTION_EXECUTION=1209600 \ _APP_MAINTENANCE_RETENTION_AUDIT=1209600 \ diff --git a/app/config/variables.php b/app/config/variables.php index 9f3bc018e..040cbadf2 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -179,7 +179,7 @@ return [ ], [ 'name' => '_APP_USAGE_TIMESERIES_INTERVAL', - 'description' => 'Interval value containing the number of seconds that the Appwrite usage process should wait before aggregating stats and syncing it to Appwrite Database from Timeseries Database. The default value is 30 seconds.', + 'description' => 'Deprecated since _ use _APP_USAGE_AGGREGATION_INTERVAL instead.', 'introduction' => '1.0.0', 'default' => '30', 'required' => false, @@ -188,7 +188,7 @@ return [ ], [ 'name' => '_APP_USAGE_DATABASE_INTERVAL', - 'description' => 'Interval value containing the number of seconds that the Appwrite usage process should wait before aggregating stats from data in Appwrite Database. The default value is 15 minutes.', + 'description' => 'Deprecated since _ use _APP_USAGE_AGGREGATION_INTERVAL instead.', 'introduction' => '1.0.0', 'default' => '900', 'required' => false, diff --git a/app/tasks/usage.php b/app/tasks/usage.php index 48876557a..1deaad230 100644 --- a/app/tasks/usage.php +++ b/app/tasks/usage.php @@ -114,44 +114,6 @@ $logError = function (Throwable $error, string $action = 'syncUsageStats') use ( Console::warning($error->getTraceAsString()); }; - -function aggregateTimeseries(UtopiaDatabase $database, InfluxDatabase $influxDB, callable $logError): void -{ - $interval = (int) App::getEnv('_APP_USAGE_TIMESERIES_INTERVAL', '30'); // 30 seconds (by default) - $usage = new TimeSeries($database, $influxDB, $logError); - - Console::loop(function () use ($interval, $usage) { - $now = date('d-m-Y H:i:s', time()); - Console::info("[{$now}] Aggregating Timeseries Usage data every {$interval} seconds"); - $loopStart = microtime(true); - - $usage->collect(); - - $loopTook = microtime(true) - $loopStart; - $now = date('d-m-Y H:i:s', time()); - Console::info("[{$now}] Aggregation took {$loopTook} seconds"); - }, $interval); -} - -function aggregateDatabase(UtopiaDatabase $database, callable $logError): void -{ - $interval = (int) App::getEnv('_APP_USAGE_DATABASE_INTERVAL', '900'); // 15 minutes (by default) - $usage = new Database($database, $logError); - $aggregrator = new Aggregator($database, $logError); - - Console::loop(function () use ($interval, $usage, $aggregrator) { - $now = date('d-m-Y H:i:s', time()); - Console::info("[{$now}] Aggregating database usage every {$interval} seconds."); - $loopStart = microtime(true); - $usage->collect(); - $aggregrator->collect(); - $loopTook = microtime(true) - $loopStart; - $now = date('d-m-Y H:i:s', time()); - - Console::info("[{$now}] Aggregation took {$loopTook} seconds"); - }, $interval); -} - $cli ->task('usage') ->param('type', 'timeseries', new WhiteList(['timeseries', 'database'])) @@ -163,14 +125,18 @@ $cli $database = getDatabase($register, '_console'); $influxDB = getInfluxDB($register); - switch ($type) { - case 'timeseries': - aggregateTimeseries($database, $influxDB, $logError); - break; - case 'database': - aggregateDatabase($database, $logError); - break; - default: - Console::error("Unsupported usage aggregation type"); - } + $interval = (int) App::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '30'); // 30 seconds (by default) + $usage = new TimeSeries($database, $influxDB, $logError); + + Console::loop(function () use ($interval, $usage) { + $now = date('d-m-Y H:i:s', time()); + Console::info("[{$now}] Aggregating Timeseries Usage data every {$interval} seconds"); + $loopStart = microtime(true); + + $usage->collect(); + + $loopTook = microtime(true) - $loopStart; + $now = date('d-m-Y H:i:s', time()); + Console::info("[{$now}] Aggregation took {$loopTook} seconds"); + }, $interval); }); diff --git a/app/views/install/compose.phtml b/app/views/install/compose.phtml index 0442d8456..07384fb0b 100644 --- a/app/views/install/compose.phtml +++ b/app/views/install/compose.phtml @@ -550,12 +550,10 @@ services: - _APP_MAINTENANCE_RETENTION_ABUSE - _APP_MAINTENANCE_RETENTION_AUDIT - appwrite-usage-timeseries: + appwrite-usage: image: /: - entrypoint: - - usage - - --type=timeseries - container_name: appwrite-usage-timeseries + entrypoint: usage + container_name: appwrite-usage <<: *x-logging restart: unless-stopped networks: @@ -573,40 +571,7 @@ services: - _APP_DB_PASS - _APP_INFLUXDB_HOST - _APP_INFLUXDB_PORT - - _APP_USAGE_TIMESERIES_INTERVAL - - _APP_USAGE_DATABASE_INTERVAL - - _APP_REDIS_HOST - - _APP_REDIS_PORT - - _APP_REDIS_USER - - _APP_REDIS_PASS - - _APP_LOGGING_PROVIDER - - _APP_LOGGING_CONFIG - - appwrite-usage-database: - image: /: - entrypoint: - - usage - - --type=database - container_name: appwrite-usage-database - <<: *x-logging - restart: unless-stopped - networks: - - appwrite - depends_on: - - influxdb - - mariadb - environment: - - _APP_ENV - - _APP_OPENSSL_KEY_V1 - - _APP_DB_HOST - - _APP_DB_PORT - - _APP_DB_SCHEMA - - _APP_DB_USER - - _APP_DB_PASS - - _APP_INFLUXDB_HOST - - _APP_INFLUXDB_PORT - - _APP_USAGE_TIMESERIES_INTERVAL - - _APP_USAGE_DATABASE_INTERVAL + - _APP_USAGE_AGGREGATION_INTERVAL - _APP_REDIS_HOST - _APP_REDIS_PORT - _APP_REDIS_USER diff --git a/docker-compose.yml b/docker-compose.yml index 5e041409b..91ec4d6b6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -567,12 +567,10 @@ services: - _APP_MAINTENANCE_RETENTION_ABUSE - _APP_MAINTENANCE_RETENTION_AUDIT - appwrite-usage-timeseries: - entrypoint: - - usage - - --type=timeseries + appwrite-usage: + entrypoint: usage <<: *x-logging - container_name: appwrite-usage-timeseries + container_name: appwrite-usage image: appwrite-dev networks: - appwrite @@ -593,52 +591,13 @@ services: - _APP_DB_PASS - _APP_INFLUXDB_HOST - _APP_INFLUXDB_PORT - - _APP_USAGE_TIMESERIES_INTERVAL - - _APP_USAGE_DATABASE_INTERVAL + - _APP_USAGE_AGGREGATION_INTERVAL - _APP_REDIS_HOST - _APP_REDIS_PORT - _APP_REDIS_USER - _APP_REDIS_PASS - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG - - # appwrite-usage-database: - # entrypoint: - # - usage - # - --type=database - # <<: *x-logging - # container_name: appwrite-usage-database - # build: - # context: . - # args: - # - DEBUG=false - # networks: - # - appwrite - # volumes: - # - ./app:/usr/src/code/app - # - ./src:/usr/src/code/src - # - ./dev:/usr/local/dev - # depends_on: - # - influxdb - # - mariadb - # environment: - # - _APP_ENV - # - _APP_OPENSSL_KEY_V1 - # - _APP_DB_HOST - # - _APP_DB_PORT - # - _APP_DB_SCHEMA - # - _APP_DB_USER - # - _APP_DB_PASS - # - _APP_INFLUXDB_HOST - # - _APP_INFLUXDB_PORT - # - _APP_USAGE_TIMESERIES_INTERVAL - # - _APP_USAGE_DATABASE_INTERVAL - # - _APP_REDIS_HOST - # - _APP_REDIS_PORT - # - _APP_REDIS_USER - # - _APP_REDIS_PASS - # - _APP_LOGGING_PROVIDER - # - _APP_LOGGING_CONFIG appwrite-schedule: entrypoint: schedule diff --git a/src/Appwrite/Usage/Calculators/Aggregator.php b/src/Appwrite/Usage/Calculators/Aggregator.php deleted file mode 100644 index f28557a06..000000000 --- a/src/Appwrite/Usage/Calculators/Aggregator.php +++ /dev/null @@ -1,231 +0,0 @@ -database->setNamespace('_' . $projectId); - - $databasesGeneralMetrics = [ - 'databases.$all.requests.create', - 'databases.$all.requests.read', - 'databases.$all.requests.update', - 'databases.$all.requests.delete', - 'collections.$all.requests.create', - 'collections.$all.requests.read', - 'collections.$all.requests.update', - 'collections.$all.requests.delete', - 'documents.$all.requests.create', - 'documents.$all.requests.read', - 'documents.$all.requests.update', - 'documents.$all.requests.delete' - ]; - - foreach ($databasesGeneralMetrics as $metric) { - $this->aggregateDailyMetric($projectId, $metric); - $this->aggregateMonthlyMetric($projectId, $metric); - } - - $databasesDatabaseMetrics = [ - 'collections.databaseId.requests.create', - 'collections.databaseId.requests.read', - 'collections.databaseId.requests.update', - 'collections.databaseId.requests.delete', - 'documents.databaseId.requests.create', - 'documents.databaseId.requests.read', - 'documents.databaseId.requests.update', - 'documents.databaseId.requests.delete', - ]; - - $this->foreachDocument($projectId, 'databases', [], function (Document $database) use ($databasesDatabaseMetrics, $projectId) { - $databaseId = $database->getId(); - foreach ($databasesDatabaseMetrics as $metric) { - $metric = str_replace('databaseId', $databaseId, $metric); - $this->aggregateDailyMetric($projectId, $metric); - $this->aggregateMonthlyMetric($projectId, $metric); - } - - $databasesCollectionMetrics = [ - 'documents.' . $databaseId . '/collectionId.requests.create', - 'documents.' . $databaseId . '/collectionId.requests.read', - 'documents.' . $databaseId . '/collectionId.requests.update', - 'documents.' . $databaseId . '/collectionId.requests.delete', - ]; - - $this->foreachDocument($projectId, 'database_' . $database->getInternalId(), [], function (Document $collection) use ($databasesCollectionMetrics, $projectId) { - $collectionId = $collection->getId(); - foreach ($databasesCollectionMetrics as $metric) { - $metric = str_replace('collectionId', $collectionId, $metric); - $this->aggregateDailyMetric($projectId, $metric); - $this->aggregateMonthlyMetric($projectId, $metric); - } - }); - }); - } - - protected function aggregateStorageMetrics(string $projectId): void - { - $this->database->setNamespace('_' . $projectId); - - $storageGeneralMetrics = [ - 'buckets.$all.requests.create', - 'buckets.$all.requests.read', - 'buckets.$all.requests.update', - 'buckets.$all.requests.delete', - 'files.$all.requests.create', - 'files.$all.requests.read', - 'files.$all.requests.update', - 'files.$all.requests.delete', - ]; - - foreach ($storageGeneralMetrics as $metric) { - $this->aggregateDailyMetric($projectId, $metric); - $this->aggregateMonthlyMetric($projectId, $metric); - } - - $storageBucketMetrics = [ - 'files.bucketId.requests.create', - 'files.bucketId.requests.read', - 'files.bucketId.requests.update', - 'files.bucketId.requests.delete', - ]; - - $this->foreachDocument($projectId, 'buckets', [], function (Document $bucket) use ($storageBucketMetrics, $projectId) { - $bucketId = $bucket->getId(); - foreach ($storageBucketMetrics as $metric) { - $metric = str_replace('bucketId', $bucketId, $metric); - $this->aggregateDailyMetric($projectId, $metric); - $this->aggregateMonthlyMetric($projectId, $metric); - } - }); - } - - protected function aggregateFunctionMetrics(string $projectId): void - { - $this->database->setNamespace('_' . $projectId); - - $functionsGeneralMetrics = [ - 'project.$all.compute.total', - 'project.$all.compute.time', - 'executions.$all.compute.total', - 'executions.$all.compute.success', - 'executions.$all.compute.failure', - 'executions.$all.compute.time', - 'builds.$all.compute.total', - 'builds.$all.compute.success', - 'builds.$all.compute.failure', - 'builds.$all.compute.time', - ]; - - foreach ($functionsGeneralMetrics as $metric) { - $this->aggregateDailyMetric($projectId, $metric); - $this->aggregateMonthlyMetric($projectId, $metric); - } - - $functionMetrics = [ - 'executions.functionId.compute.total', - 'executions.functionId.compute.success', - 'executions.functionId.compute.failure', - 'executions.functionId.compute.time', - 'builds.functionId.compute.total', - 'builds.functionId.compute.success', - 'builds.functionId.compute.failure', - 'builds.functionId.compute.time', - ]; - - $this->foreachDocument($projectId, 'functions', [], function (Document $function) use ($functionMetrics, $projectId) { - $functionId = $function->getId(); - foreach ($functionMetrics as $metric) { - $metric = str_replace('functionId', $functionId, $metric); - $this->aggregateDailyMetric($projectId, $metric); - $this->aggregateMonthlyMetric($projectId, $metric); - } - }); - } - - protected function aggregateUsersMetrics(string $projectId): void - { - $metrics = [ - 'users.$all.requests.create', - 'users.$all.requests.read', - 'users.$all.requests.update', - 'users.$all.requests.delete', - 'sessions.$all.requests.create', - 'sessions.$all.requests.delete' - ]; - - foreach ($metrics as $metric) { - $this->aggregateDailyMetric($projectId, $metric); - $this->aggregateMonthlyMetric($projectId, $metric); - } - } - - protected function aggregateGeneralMetrics(string $projectId): void - { - $this->aggregateDailyMetric($projectId, 'project.$all.network.requests'); - $this->aggregateDailyMetric($projectId, 'project.$all.network.bandwidth'); - $this->aggregateDailyMetric($projectId, 'project.$all.network.inbound'); - $this->aggregateDailyMetric($projectId, 'project.$all.network.outbound'); - $this->aggregateMonthlyMetric($projectId, 'project.$all.network.requests'); - $this->aggregateMonthlyMetric($projectId, 'project.$all.network.bandwidth'); - $this->aggregateMonthlyMetric($projectId, 'project.$all.network.inbound'); - $this->aggregateMonthlyMetric($projectId, 'project.$all.network.outbound'); - } - - protected function aggregateDailyMetric(string $projectId, string $metric): void - { - $beginOfDay = DateTime::createFromFormat('Y-m-d\TH:i:s.v', \date('Y-m-d\T00:00:00.000'))->format(DateTime::RFC3339); - $endOfDay = DateTime::createFromFormat('Y-m-d\TH:i:s.v', \date('Y-m-d\T23:59:59.999'))->format(DateTime::RFC3339); - - $this->database->setNamespace('_' . $projectId); - $value = (int) $this->database->sum('stats', 'value', [ - Query::equal('metric', [$metric]), - Query::equal('period', ['1h']), - Query::greaterThanEqual('time', $beginOfDay), - Query::lessThanEqual('time', $endOfDay), - ]); - $this->createOrUpdateMetric($projectId, $metric, '1d', $beginOfDay, $value); - } - - protected function aggregateMonthlyMetric(string $projectId, string $metric): void - { - $beginOfMonth = DateTime::createFromFormat('Y-m-d\TH:i:s.v', \date('Y-m-01\T00:00:00.000'))->format(DateTime::RFC3339); - $endOfMonth = DateTime::createFromFormat('Y-m-d\TH:i:s.v', \date('Y-m-t\T23:59:59.999'))->format(DateTime::RFC3339); - $this->database->setNamespace('_' . $projectId); - $value = (int) $this->database->sum('stats', 'value', [ - Query::equal('metric', [$metric]), - Query::equal('period', ['1d']), - Query::greaterThanEqual('time', $beginOfMonth), - Query::lessThanEqual('time', $endOfMonth), - ]); - $this->createOrUpdateMetric($projectId, $metric, '1mo', $beginOfMonth, $value); - } - - /** - * Collect Stats - * Collect all database related stats - * - * @return void - */ - public function collect(): void - { - $this->foreachDocument('console', 'projects', [], function (Document $project) { - $projectId = $project->getInternalId(); - - // Aggregate new metrics from already collected usage metrics - // for lower time period (1day and 1 month metric from 30 minute metrics) - $this->aggregateGeneralMetrics($projectId); - $this->aggregateFunctionMetrics($projectId); - $this->aggregateDatabaseMetrics($projectId); - $this->aggregateStorageMetrics($projectId); - $this->aggregateUsersMetrics($projectId); - }); - } -} diff --git a/src/Appwrite/Usage/Calculators/Database.php b/src/Appwrite/Usage/Calculators/Database.php deleted file mode 100644 index d760e6a0c..000000000 --- a/src/Appwrite/Usage/Calculators/Database.php +++ /dev/null @@ -1,360 +0,0 @@ - '1h', - 'multiplier' => 3600, - ], - [ - 'key' => '1d', - 'multiplier' => 86400, - ], - ]; - - public function __construct(UtopiaDatabase $database, callable $errorHandler = null) - { - $this->database = $database; - $this->errorHandler = $errorHandler; - } - - /** - * Create Per Period Metric - * - * Create given metric for each defined period - * - * @param string $projectId - * @param string $metric - * @param int $value - * @param bool $monthly - * @return void - * @throws Authorization - * @throws Structure - */ - protected function createPerPeriodMetric(string $projectId, string $metric, int $value, bool $monthly = false): void - { - foreach ($this->periods as $options) { - $period = $options['key']; - $date = new \DateTime(); - if ($period === '1h') { - $minutes = $date->format('i') >= '30' ? "30" : "00"; - $time = $date->format('Y-m-d H:' . $minutes . ':00'); - } elseif ($period === '1d') { - $time = $date->format('Y-m-d 00:00:00'); - } else { - throw new Exception("Period type not found", 500); - } - $this->createOrUpdateMetric($projectId, $metric, $period, $time, $value); - } - - // Required for billing - if ($monthly) { - $time = DateTime::createFromFormat('Y-m-d\TH:i:s.v', \date('Y-m-01\T00:00:00.000'))->format(DateTime::RFC3339); - $this->createOrUpdateMetric($projectId, $metric, '1mo', $time, $value); - } - } - - /** - * Create or Update Metric - * - * Create or update each metric in the stats collection for the given project - * - * @param string $projectId - * @param string $metric - * @param string $period - * @param string $time - * @param int $value - * - * @return void - * @throws Authorization - * @throws Structure - */ - protected function createOrUpdateMetric(string $projectId, string $metric, string $period, string $time, int $value): void - { - $id = \md5("{$time}_{$period}_{$metric}"); - $this->database->setNamespace('_' . $projectId); - - try { - $document = $this->database->getDocument('stats', $id); - if ($document->isEmpty()) { - $this->database->createDocument('stats', new Document([ - '$id' => $id, - 'period' => $period, - 'time' => $time, - 'metric' => $metric, - 'value' => $value, - 'type' => 2, // these are cumulative metrics - ])); - } else { - $this->database->updateDocument( - 'stats', - $document->getId(), - $document->setAttribute('value', $value) - ); - } - } catch (\Exception$e) { // if projects are deleted this might fail - if (is_callable($this->errorHandler)) { - call_user_func($this->errorHandler, $e, "sync_project_{$projectId}_metric_{$metric}"); - } else { - throw $e; - } - } - } - - /** - * Foreach Document - * - * Call provided callback for each document in the collection - * - * @param string $projectId - * @param string $collection - * @param array $queries - * @param callable $callback - * - * @return void - * @throws Exception - */ - protected function foreachDocument(string $projectId, string $collection, array $queries, callable $callback): void - { - $limit = 50; - $results = []; - $sum = $limit; - $latestDocument = null; - $this->database->setNamespace('_' . $projectId); - - while ($sum === $limit) { - try { - $paginationQueries = [Query::limit($limit)]; - if ($latestDocument !== null) { - $paginationQueries[] = Query::cursorAfter($latestDocument); - } - $results = $this->database->find($collection, \array_merge($paginationQueries, $queries)); - } catch (\Exception $e) { - if (is_callable($this->errorHandler)) { - call_user_func($this->errorHandler, $e, "fetch_documents_project_{$projectId}_collection_{$collection}"); - return; - } else { - throw $e; - } - } - if (empty($results)) { - return; - } - - $sum = count($results); - - foreach ($results as $document) { - if (is_callable($callback)) { - $callback($document); - } - } - $latestDocument = $results[array_key_last($results)]; - } - } - - /** - * Sum - * - * Calculate sum of an attribute of documents in collection - * - * @param string $projectId - * @param string $collection - * @param string $attribute - * @param string|null $metric - * @param int $multiplier - * @return int - * @throws Exception - */ - private function sum(string $projectId, string $collection, string $attribute, string $metric = null, int $multiplier = 1): int - { - $this->database->setNamespace('_' . $projectId); - - try { - $sum = $this->database->sum($collection, $attribute); - $sum = (int) ($sum * $multiplier); - - if (!is_null($metric)) { - $this->createPerPeriodMetric($projectId, $metric, $sum); - } - return $sum; - } catch (Exception $e) { - if (is_callable($this->errorHandler)) { - call_user_func($this->errorHandler, $e, "fetch_sum_project_{$projectId}_collection_{$collection}"); - } else { - throw $e; - } - } - return 0; - } - - /** - * Count - * - * Count number of documents in collection - * - * @param string $projectId - * @param string $collection - * @param ?string $metric - * - * @return int - * @throws Exception - */ - private function count(string $projectId, string $collection, ?string $metric = null): int - { - $this->database->setNamespace('_' . $projectId); - - try { - $count = $this->database->count($collection); - if (!is_null($metric)) { - $this->createPerPeriodMetric($projectId, (string) $metric, $count); - } - return $count; - } catch (Exception $e) { - if (is_callable($this->errorHandler)) { - call_user_func($this->errorHandler, $e, "fetch_count_project_{$projectId}_collection_{$collection}"); - } else { - throw $e; - } - } - return 0; - } - - /** - * Deployments Total - * - * Total sum of storage used by deployments - * - * @param string $projectId - * - * @return int - * @throws Exception - */ - private function deploymentsTotal(string $projectId): int - { - return $this->sum($projectId, 'deployments', 'size', 'deployments.$all.storage.size'); - } - - /** - * Users Stats - * - * Metric: users.count - * - * @param string $projectId - * - * @return void - * @throws Exception - */ - private function usersStats(string $projectId): void - { - $this->count($projectId, 'users', 'users.$all.count.total'); - } - - /** - * Storage Stats - * - * Metrics: buckets.$all.count.total, files.$all.count.total, files.bucketId,count.total, - * files.$all.storage.size, files.bucketId.storage.size, project.$all.storage.size - * - * @param string $projectId - * - * @return void - * @throws Authorization - * @throws Structure - */ - private function storageStats(string $projectId): void - { - $projectFilesTotal = 0; - $projectFilesCount = 0; - - $metric = 'buckets.$all.count.total'; - $this->count($projectId, 'buckets', $metric); - - $this->foreachDocument($projectId, 'buckets', [], function ($bucket) use (&$projectFilesCount, &$projectFilesTotal, $projectId,) { - $metric = "files.{$bucket->getId()}.count.total"; - $count = $this->count($projectId, 'bucket_' . $bucket->getInternalId(), $metric); - $projectFilesCount += $count; - - $metric = "files.{$bucket->getId()}.storage.size"; - $sum = $this->sum($projectId, 'bucket_' . $bucket->getInternalId(), 'sizeOriginal', $metric); - $projectFilesTotal += $sum; - }); - - $this->createPerPeriodMetric($projectId, 'files.$all.count.total', $projectFilesCount); - $this->createPerPeriodMetric($projectId, 'files.$all.storage.size', $projectFilesTotal); - - $deploymentsTotal = $this->deploymentsTotal($projectId); - $this->createPerPeriodMetric($projectId, 'project.$all.storage.size', $projectFilesTotal + $deploymentsTotal); - } - - /** - * Database Stats - * - * Collect all database stats - * Metrics: databases.$all.count.total, collections.$all.count.total, collections.databaseId.count.total, - * documents.$all.count.all, documents.databaseId.count.total, documents.databaseId/collectionId.count.total - * - * @param string $projectId - * - * @return void - * @throws Authorization - * @throws Structure - */ - private function databaseStats(string $projectId): void - { - $projectDocumentsCount = 0; - $projectCollectionsCount = 0; - - $this->count($projectId, 'databases', 'databases.$all.count.total'); - - $this->foreachDocument($projectId, 'databases', [], function ($database) use (&$projectDocumentsCount, &$projectCollectionsCount, $projectId) { - $metric = "collections.{$database->getId()}.count.total"; - $count = $this->count($projectId, 'database_' . $database->getInternalId(), $metric); - $projectCollectionsCount += $count; - $databaseDocumentsCount = 0; - - $this->foreachDocument($projectId, 'database_' . $database->getInternalId(), [], function ($collection) use (&$projectDocumentsCount, &$databaseDocumentsCount, $projectId, $database) { - $metric = "documents.{$database->getId()}/{$collection->getId()}.count.total"; - - $count = $this->count($projectId, 'database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $metric); - $projectDocumentsCount += $count; - $databaseDocumentsCount += $count; - }); - - $this->createPerPeriodMetric($projectId, "documents.{$database->getId()}.count.total", $databaseDocumentsCount); - }); - - $this->createPerPeriodMetric($projectId, 'collections.$all.count.total', $projectCollectionsCount); - $this->createPerPeriodMetric($projectId, 'documents.$all.count.total', $projectDocumentsCount); - } - - /** - * Collect Stats - * - * Collect all database related stats - * - * @return void - * @throws Exception - */ - public function collect(): void - { - $this->foreachDocument('console', 'projects', [], function (Document $project) { - $projectId = $project->getInternalId(); - - $this->usersStats($projectId); - $this->databaseStats($projectId); - $this->storageStats($projectId); - }); - } -} From bde7fe21c456ec478cd56c65bdad568dcb8523f5 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 23 Oct 2022 07:03:50 +0000 Subject: [PATCH 50/69] remove unused param --- app/tasks/usage.php | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/app/tasks/usage.php b/app/tasks/usage.php index 1deaad230..b28be1de1 100644 --- a/app/tasks/usage.php +++ b/app/tasks/usage.php @@ -2,10 +2,6 @@ global $cli, $register; -use Appwrite\Stats\Usage; -use Appwrite\Stats\UsageDB; -use Appwrite\Usage\Calculators\Aggregator; -use Appwrite\Usage\Calculators\Database; use Appwrite\Usage\Calculators\TimeSeries; use InfluxDB\Database as InfluxDatabase; use Utopia\App; @@ -116,9 +112,8 @@ $logError = function (Throwable $error, string $action = 'syncUsageStats') use ( $cli ->task('usage') - ->param('type', 'timeseries', new WhiteList(['timeseries', 'database'])) ->desc('Schedules syncing data from influxdb to Appwrite console db') - ->action(function (string $type) use ($register, $logError) { + ->action(function () use ($register, $logError) { Console::title('Usage Aggregation V1'); Console::success(APP_NAME . ' usage aggregation process v1 has started'); From 8c3cfa877b41a6722595b1983a7423142a38d761 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 23 Oct 2022 07:18:52 +0000 Subject: [PATCH 51/69] open api schema validator changes --- tests/e2e/General/HTTPTest.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/e2e/General/HTTPTest.php b/tests/e2e/General/HTTPTest.php index fd5ac43ea..1be23361f 100644 --- a/tests/e2e/General/HTTPTest.php +++ b/tests/e2e/General/HTTPTest.php @@ -182,7 +182,8 @@ class HTTPTest extends Scope $response['body'] = json_decode($response['body'], true); $this->assertEquals(200, $response['headers']['status-code']); - $this->assertTrue(empty($response['body'])); + // looks like recent change in the validator + $this->assertTrue(empty($response['body']['schemaValidationMessages'])); } } From c8b53012679766f5748f1e7d774cd805280f3da5 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 23 Oct 2022 07:29:13 +0000 Subject: [PATCH 52/69] update maintenance to delete only hourly metrics older than 3 months --- app/tasks/maintenance.php | 11 +++++------ app/workers/deletes.php | 10 ++-------- src/Appwrite/Event/Delete.php | 14 -------------- 3 files changed, 7 insertions(+), 28 deletions(-) diff --git a/app/tasks/maintenance.php b/app/tasks/maintenance.php index e19870289..66b396f58 100644 --- a/app/tasks/maintenance.php +++ b/app/tasks/maintenance.php @@ -78,11 +78,10 @@ $cli ->trigger(); } - function notifyDeleteUsageStats(int $interval1h, int $interval1d) + function notifyDeleteUsageStats(int $interval1h) { (new Delete()) ->setType(DELETE_TYPE_USAGE) - ->setDateTime1d(DateTime::addSeconds(new \DateTime(), -1 * $interval1d)) ->setDateTime1h(DateTime::addSeconds(new \DateTime(), -1 * $interval1h)) ->trigger(); } @@ -144,11 +143,11 @@ $cli $executionLogsRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_EXECUTION', '1209600'); $auditLogRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT', '1209600'); $abuseLogsRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_ABUSE', '86400'); - $usageStatsRetention1h = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_USAGE_30M', '129600'); //36 hours - $usageStatsRetention1d = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_USAGE_1D', '8640000'); // 100 days + $usageStatsRetention1h = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_USAGE_1H', '8640000'); //100 days + $cacheRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_CACHE', '2592000'); // 30 days - Console::loop(function () use ($interval, $executionLogsRetention, $abuseLogsRetention, $auditLogRetention, $usageStatsRetention1h, $usageStatsRetention1d, $cacheRetention) { + Console::loop(function () use ($interval, $executionLogsRetention, $abuseLogsRetention, $auditLogRetention, $usageStatsRetention1h, $cacheRetention) { $database = getConsoleDB(); $time = DateTime::now(); @@ -157,7 +156,7 @@ $cli notifyDeleteExecutionLogs($executionLogsRetention); notifyDeleteAbuseLogs($abuseLogsRetention); notifyDeleteAuditLogs($auditLogRetention); - notifyDeleteUsageStats($usageStatsRetention1h, $usageStatsRetention1d); + notifyDeleteUsageStats($usageStatsRetention1h); notifyDeleteConnections(); notifyDeleteExpiredSessions(); renewCertificates($database); diff --git a/app/workers/deletes.php b/app/workers/deletes.php index a5db6db5f..737dc96bb 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -217,16 +217,10 @@ class DeletesV1 extends Worker * @param string $datetime1d * @param string $datetime1h */ - protected function deleteUsageStats(string $datetime1d, string $datetime1h) + protected function deleteUsageStats(string $datetime1h) { - $this->deleteForProjectIds(function (string $projectId) use ($datetime1d, $datetime1h) { + $this->deleteForProjectIds(function (string $projectId) use ($datetime1h) { $dbForProject = $this->getProjectDB($projectId); - // Delete Usage stats - $this->deleteByGroup('stats', [ - Query::lessThan('time', $datetime1d), - Query::equal('period', ['1d']), - ], $dbForProject); - $this->deleteByGroup('stats', [ Query::lessThan('time', $datetime1h), Query::equal('period', ['1h']), diff --git a/src/Appwrite/Event/Delete.php b/src/Appwrite/Event/Delete.php index cd2461c6e..51d2bfa47 100644 --- a/src/Appwrite/Event/Delete.php +++ b/src/Appwrite/Event/Delete.php @@ -12,7 +12,6 @@ class Delete extends Event protected ?string $resource = null; protected ?string $datetime = null; protected ?string $dateTime1h = null; - protected ?string $dateTime1d = null; public function __construct() @@ -55,18 +54,6 @@ class Delete extends Event return $this; } - /** - * Set datetime for 1 day interval. - * - * @param string $datetime - * @return self - */ - public function setDateTime1d(string $datetime): self - { - $this->dateTime1d = $datetime; - return $this; - } - /** * Sets datetime for 1h interval. * @@ -140,7 +127,6 @@ class Delete extends Event 'document' => $this->document, 'resource' => $this->resource, 'datetime' => $this->datetime, - 'dateTime1d' => $this->dateTime1d, 'dateTime1h' => $this->dateTime1h, ]); } From a0a0ef23868f1a26a6c3d49a00c22cc9b1d4939c Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Fri, 28 Oct 2022 08:40:04 +0000 Subject: [PATCH 53/69] update docblock comments and naming --- app/tasks/maintenance.php | 10 +++--- app/workers/deletes.php | 10 +++--- src/Appwrite/Event/Delete.php | 8 ++--- src/Appwrite/Usage/Calculators/TimeSeries.php | 35 ++++++++++++++++++- src/Appwrite/Usage/Stats.php | 2 ++ 5 files changed, 50 insertions(+), 15 deletions(-) diff --git a/app/tasks/maintenance.php b/app/tasks/maintenance.php index 66b396f58..e7c6063f6 100644 --- a/app/tasks/maintenance.php +++ b/app/tasks/maintenance.php @@ -78,11 +78,11 @@ $cli ->trigger(); } - function notifyDeleteUsageStats(int $interval1h) + function notifyDeleteUsageStats(int $hourlyRetentionInterval) { (new Delete()) ->setType(DELETE_TYPE_USAGE) - ->setDateTime1h(DateTime::addSeconds(new \DateTime(), -1 * $interval1h)) + ->setHourlyUsageRetentionDatetime(DateTime::addSeconds(new \DateTime(), -1 * $hourlyRetentionInterval)) ->trigger(); } @@ -143,11 +143,11 @@ $cli $executionLogsRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_EXECUTION', '1209600'); $auditLogRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT', '1209600'); $abuseLogsRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_ABUSE', '86400'); - $usageStatsRetention1h = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_USAGE_1H', '8640000'); //100 days + $hourlyUsageStatsRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_HOURLY_USAGE', '8640000'); //100 days $cacheRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_CACHE', '2592000'); // 30 days - Console::loop(function () use ($interval, $executionLogsRetention, $abuseLogsRetention, $auditLogRetention, $usageStatsRetention1h, $cacheRetention) { + Console::loop(function () use ($interval, $executionLogsRetention, $abuseLogsRetention, $auditLogRetention, $hourlyUsageStatsRetention, $cacheRetention) { $database = getConsoleDB(); $time = DateTime::now(); @@ -156,7 +156,7 @@ $cli notifyDeleteExecutionLogs($executionLogsRetention); notifyDeleteAbuseLogs($abuseLogsRetention); notifyDeleteAuditLogs($auditLogRetention); - notifyDeleteUsageStats($usageStatsRetention1h); + notifyDeleteUsageStats($hourlyUsageStatsRetention); notifyDeleteConnections(); notifyDeleteExpiredSessions(); renewCertificates($database); diff --git a/app/workers/deletes.php b/app/workers/deletes.php index 737dc96bb..44ae1136a 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -105,7 +105,7 @@ class DeletesV1 extends Worker break; case DELETE_TYPE_USAGE: - $this->deleteUsageStats($this->args['dateTime1d'], $this->args['dateTime1h']); + $this->deleteUsageStats($this->args['dateTime1d'], $this->args['hourlyUsageRetentionDatetime']); break; case DELETE_TYPE_CACHE_BY_RESOURCE: @@ -215,14 +215,14 @@ class DeletesV1 extends Worker /** * @param string $datetime1d - * @param string $datetime1h + * @param string $hourlyUsageRetentionDatetime */ - protected function deleteUsageStats(string $datetime1h) + protected function deleteUsageStats(string $hourlyUsageRetentionDatetime) { - $this->deleteForProjectIds(function (string $projectId) use ($datetime1h) { + $this->deleteForProjectIds(function (string $projectId) use ($hourlyUsageRetentionDatetime) { $dbForProject = $this->getProjectDB($projectId); $this->deleteByGroup('stats', [ - Query::lessThan('time', $datetime1h), + Query::lessThan('time', $hourlyUsageRetentionDatetime), Query::equal('period', ['1h']), ], $dbForProject); }); diff --git a/src/Appwrite/Event/Delete.php b/src/Appwrite/Event/Delete.php index 51d2bfa47..90a77d489 100644 --- a/src/Appwrite/Event/Delete.php +++ b/src/Appwrite/Event/Delete.php @@ -11,7 +11,7 @@ class Delete extends Event protected ?Document $document = null; protected ?string $resource = null; protected ?string $datetime = null; - protected ?string $dateTime1h = null; + protected ?string $hourlyUsageRetentionDatetime = null; public function __construct() @@ -60,9 +60,9 @@ class Delete extends Event * @param string $datetime * @return self */ - public function setDateTime1h(string $datetime): self + public function setHourlyUsageRetentionDatetime(string $datetime): self { - $this->dateTime1h = $datetime; + $this->hourlyUsageRetentionDatetime = $datetime; return $this; } @@ -127,7 +127,7 @@ class Delete extends Event 'document' => $this->document, 'resource' => $this->resource, 'datetime' => $this->datetime, - 'dateTime1h' => $this->dateTime1h, + 'hourlyUsageRetentionDatetime' => $this->hourlyUsageRetentionDatetime, ]); } } diff --git a/src/Appwrite/Usage/Calculators/TimeSeries.php b/src/Appwrite/Usage/Calculators/TimeSeries.php index c58722838..6993da163 100644 --- a/src/Appwrite/Usage/Calculators/TimeSeries.php +++ b/src/Appwrite/Usage/Calculators/TimeSeries.php @@ -10,10 +10,38 @@ use DateTime; class TimeSeries extends Calculator { + /** + * InfluxDB + * + * @var InfluxDatabase + */ protected InfluxDatabase $influxDB; + + /** + * Utopia Database + * + * @var Database + */ protected Database $database; + + /** + * Error Handler Callback + * + * @var callable + */ protected $errorHandler; + + /** + * Latest times for metric that was synced to the database + * + * @var array + */ private array $latestTime = []; + + /** + * Periods the metrics are collected for + * @var array + */ protected array $periods = [ [ 'key' => '1h', @@ -24,7 +52,12 @@ class TimeSeries extends Calculator 'startTime' => '-30 days' ] ]; - // all the mertics that we are collecting + + /** + * All the metrics that we are collecting + * + * @var array + */ protected array $metrics = [ 'project.$all.network.requests' => [ 'table' => 'appwrite_usage_project_{scope}_network_requests', diff --git a/src/Appwrite/Usage/Stats.php b/src/Appwrite/Usage/Stats.php index ba68d9ddf..510584c87 100644 --- a/src/Appwrite/Usage/Stats.php +++ b/src/Appwrite/Usage/Stats.php @@ -76,6 +76,8 @@ class Stats /** * Submit data to StatsD. + * Send various metrics to StatsD based on the parameters that are set + * @return void */ public function submit(): void { From 6d7b30d77e7155ea95a0ff42b1967db503529364 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Fri, 28 Oct 2022 09:08:58 +0000 Subject: [PATCH 54/69] fix formatting issues --- src/Appwrite/Usage/Calculators/TimeSeries.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Usage/Calculators/TimeSeries.php b/src/Appwrite/Usage/Calculators/TimeSeries.php index 6993da163..20308684d 100644 --- a/src/Appwrite/Usage/Calculators/TimeSeries.php +++ b/src/Appwrite/Usage/Calculators/TimeSeries.php @@ -55,7 +55,7 @@ class TimeSeries extends Calculator /** * All the metrics that we are collecting - * + * * @var array */ protected array $metrics = [ From 4c5273b8b227c5a9bd114461ac61c1d8555cf900 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Fri, 28 Oct 2022 09:16:36 +0000 Subject: [PATCH 55/69] document the environment variable usage stats retention --- .env | 1 + Dockerfile | 1 + app/config/variables.php | 11 ++++++++++- app/views/install/compose.phtml | 2 ++ docker-compose.yml | 2 ++ 5 files changed, 16 insertions(+), 1 deletion(-) diff --git a/.env b/.env index ea5738cc9..02e645d9b 100644 --- a/.env +++ b/.env @@ -77,6 +77,7 @@ _APP_MAINTENANCE_RETENTION_EXECUTION=1209600 _APP_MAINTENANCE_RETENTION_ABUSE=86400 _APP_MAINTENANCE_RETENTION_AUDIT=1209600 _APP_USAGE_AGGREGATION_INTERVAL=5 +_APP_MAINTENANCE_RETENTION_HOURLY_USAGE=8640000 _APP_USAGE_STATS=enabled _APP_LOGGING_PROVIDER= _APP_LOGGING_CONFIG= diff --git a/Dockerfile b/Dockerfile index 2635b97e0..9f0d71c7b 100755 --- a/Dockerfile +++ b/Dockerfile @@ -252,6 +252,7 @@ ENV _APP_SERVER=swoole \ _APP_MAINTENANCE_RETENTION_AUDIT=1209600 \ # 1 Day = 86400 s _APP_MAINTENANCE_RETENTION_ABUSE=86400 \ + _APP_MAINTENANCE_RETENTION_HOURLY_USAGE=8640000 \ _APP_MAINTENANCE_INTERVAL=86400 \ _APP_LOGGING_PROVIDER= \ _APP_LOGGING_CONFIG= diff --git a/app/config/variables.php b/app/config/variables.php index 040cbadf2..d00e0e10c 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -857,7 +857,16 @@ return [ 'required' => false, 'question' => '', 'filter' => '' - ] + ], + [ + 'name' => '_APP_MAINTENANCE_RETENTION_HOURLY_USAGE', + 'description' => 'The maximum duration (in seconds) upto which to retain hourly usage metrics. The default value is 8640000 seconds (100 days).', + 'introduction' => '', + 'default' => '8640000', + 'required' => false, + 'question' => '', + 'filter' => '' + ], ], ], ]; diff --git a/app/views/install/compose.phtml b/app/views/install/compose.phtml index 07384fb0b..e180cfccb 100644 --- a/app/views/install/compose.phtml +++ b/app/views/install/compose.phtml @@ -150,6 +150,7 @@ services: - _APP_MAINTENANCE_RETENTION_CACHE - _APP_MAINTENANCE_RETENTION_ABUSE - _APP_MAINTENANCE_RETENTION_AUDIT + - _APP_MAINTENANCE_RETENTION_HOURLY_USAGE - _APP_SMS_PROVIDER - _APP_SMS_FROM @@ -549,6 +550,7 @@ services: - _APP_MAINTENANCE_RETENTION_CACHE - _APP_MAINTENANCE_RETENTION_ABUSE - _APP_MAINTENANCE_RETENTION_AUDIT + - _APP_MAINTENANCE_RETENTION_HOURLY_USAGE appwrite-usage: image: /: diff --git a/docker-compose.yml b/docker-compose.yml index 53bb7dee7..25b455c36 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -171,6 +171,7 @@ services: - _APP_MAINTENANCE_RETENTION_CACHE - _APP_MAINTENANCE_RETENTION_ABUSE - _APP_MAINTENANCE_RETENTION_AUDIT + - _APP_MAINTENANCE_RETENTION_HOURLY_USAGE - _APP_SMS_PROVIDER - _APP_SMS_FROM @@ -602,6 +603,7 @@ services: - _APP_MAINTENANCE_RETENTION_CACHE - _APP_MAINTENANCE_RETENTION_ABUSE - _APP_MAINTENANCE_RETENTION_AUDIT + - _APP_MAINTENANCE_RETENTION_HOURLY_USAGE appwrite-usage: entrypoint: usage From 60f2d10a50afc6dc1eae394ba9c323dc19346b83 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Fri, 28 Oct 2022 09:54:56 +0000 Subject: [PATCH 56/69] update variable namings --- .env | 2 +- Dockerfile | 2 +- app/config/variables.php | 2 +- app/tasks/maintenance.php | 10 +++++----- app/views/install/compose.phtml | 4 ++-- docker-compose.yml | 4 ++-- src/Appwrite/Event/Delete.php | 2 +- 7 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.env b/.env index 02e645d9b..c616312e5 100644 --- a/.env +++ b/.env @@ -77,7 +77,7 @@ _APP_MAINTENANCE_RETENTION_EXECUTION=1209600 _APP_MAINTENANCE_RETENTION_ABUSE=86400 _APP_MAINTENANCE_RETENTION_AUDIT=1209600 _APP_USAGE_AGGREGATION_INTERVAL=5 -_APP_MAINTENANCE_RETENTION_HOURLY_USAGE=8640000 +_APP_MAINTENANCE_RETENTION_USAGE_HOURLY=8640000 _APP_USAGE_STATS=enabled _APP_LOGGING_PROVIDER= _APP_LOGGING_CONFIG= diff --git a/Dockerfile b/Dockerfile index 9f0d71c7b..725fcaa46 100755 --- a/Dockerfile +++ b/Dockerfile @@ -252,7 +252,7 @@ ENV _APP_SERVER=swoole \ _APP_MAINTENANCE_RETENTION_AUDIT=1209600 \ # 1 Day = 86400 s _APP_MAINTENANCE_RETENTION_ABUSE=86400 \ - _APP_MAINTENANCE_RETENTION_HOURLY_USAGE=8640000 \ + _APP_MAINTENANCE_RETENTION_USAGE_HOURLY=8640000 \ _APP_MAINTENANCE_INTERVAL=86400 \ _APP_LOGGING_PROVIDER= \ _APP_LOGGING_CONFIG= diff --git a/app/config/variables.php b/app/config/variables.php index d00e0e10c..dbfabaab7 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -859,7 +859,7 @@ return [ 'filter' => '' ], [ - 'name' => '_APP_MAINTENANCE_RETENTION_HOURLY_USAGE', + 'name' => '_APP_MAINTENANCE_RETENTION_USAGE_HOURLY', 'description' => 'The maximum duration (in seconds) upto which to retain hourly usage metrics. The default value is 8640000 seconds (100 days).', 'introduction' => '', 'default' => '8640000', diff --git a/app/tasks/maintenance.php b/app/tasks/maintenance.php index e7c6063f6..92f260ce4 100644 --- a/app/tasks/maintenance.php +++ b/app/tasks/maintenance.php @@ -78,11 +78,11 @@ $cli ->trigger(); } - function notifyDeleteUsageStats(int $hourlyRetentionInterval) + function notifyDeleteUsageStats(int $usageStatsRetentionHourly) { (new Delete()) ->setType(DELETE_TYPE_USAGE) - ->setHourlyUsageRetentionDatetime(DateTime::addSeconds(new \DateTime(), -1 * $hourlyRetentionInterval)) + ->setUsageRetentionHourlyDateTime(DateTime::addSeconds(new \DateTime(), -1 * $usageStatsRetentionHourly)) ->trigger(); } @@ -143,11 +143,11 @@ $cli $executionLogsRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_EXECUTION', '1209600'); $auditLogRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT', '1209600'); $abuseLogsRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_ABUSE', '86400'); - $hourlyUsageStatsRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_HOURLY_USAGE', '8640000'); //100 days + $usageStatsRetentionHourly = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_USAGE_HOURLY', '8640000'); //100 days $cacheRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_CACHE', '2592000'); // 30 days - Console::loop(function () use ($interval, $executionLogsRetention, $abuseLogsRetention, $auditLogRetention, $hourlyUsageStatsRetention, $cacheRetention) { + Console::loop(function () use ($interval, $executionLogsRetention, $abuseLogsRetention, $auditLogRetention, $usageStatsRetentionHourly, $cacheRetention) { $database = getConsoleDB(); $time = DateTime::now(); @@ -156,7 +156,7 @@ $cli notifyDeleteExecutionLogs($executionLogsRetention); notifyDeleteAbuseLogs($abuseLogsRetention); notifyDeleteAuditLogs($auditLogRetention); - notifyDeleteUsageStats($hourlyUsageStatsRetention); + notifyDeleteUsageStats($usageStatsRetentionHourly); notifyDeleteConnections(); notifyDeleteExpiredSessions(); renewCertificates($database); diff --git a/app/views/install/compose.phtml b/app/views/install/compose.phtml index e180cfccb..e14e434a8 100644 --- a/app/views/install/compose.phtml +++ b/app/views/install/compose.phtml @@ -150,7 +150,7 @@ services: - _APP_MAINTENANCE_RETENTION_CACHE - _APP_MAINTENANCE_RETENTION_ABUSE - _APP_MAINTENANCE_RETENTION_AUDIT - - _APP_MAINTENANCE_RETENTION_HOURLY_USAGE + - _APP_MAINTENANCE_RETENTION_USAGE_HOURLY - _APP_SMS_PROVIDER - _APP_SMS_FROM @@ -550,7 +550,7 @@ services: - _APP_MAINTENANCE_RETENTION_CACHE - _APP_MAINTENANCE_RETENTION_ABUSE - _APP_MAINTENANCE_RETENTION_AUDIT - - _APP_MAINTENANCE_RETENTION_HOURLY_USAGE + - _APP_MAINTENANCE_RETENTION_USAGE_HOURLY appwrite-usage: image: /: diff --git a/docker-compose.yml b/docker-compose.yml index 25b455c36..18ea38f65 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -171,7 +171,7 @@ services: - _APP_MAINTENANCE_RETENTION_CACHE - _APP_MAINTENANCE_RETENTION_ABUSE - _APP_MAINTENANCE_RETENTION_AUDIT - - _APP_MAINTENANCE_RETENTION_HOURLY_USAGE + - _APP_MAINTENANCE_RETENTION_USAGE_HOURLY - _APP_SMS_PROVIDER - _APP_SMS_FROM @@ -603,7 +603,7 @@ services: - _APP_MAINTENANCE_RETENTION_CACHE - _APP_MAINTENANCE_RETENTION_ABUSE - _APP_MAINTENANCE_RETENTION_AUDIT - - _APP_MAINTENANCE_RETENTION_HOURLY_USAGE + - _APP_MAINTENANCE_RETENTION_USAGE_HOURLY appwrite-usage: entrypoint: usage diff --git a/src/Appwrite/Event/Delete.php b/src/Appwrite/Event/Delete.php index 90a77d489..d1519121a 100644 --- a/src/Appwrite/Event/Delete.php +++ b/src/Appwrite/Event/Delete.php @@ -60,7 +60,7 @@ class Delete extends Event * @param string $datetime * @return self */ - public function setHourlyUsageRetentionDatetime(string $datetime): self + public function setUsageRetentionHourlyDateTime(string $datetime): self { $this->hourlyUsageRetentionDatetime = $datetime; return $this; From 80972aa4ce055b37002e6b7acc4b660d7cfed6e2 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 30 Oct 2022 05:07:46 +0000 Subject: [PATCH 57/69] update variables descriptions --- app/config/variables.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/config/variables.php b/app/config/variables.php index dbfabaab7..40bb99f8f 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -170,8 +170,8 @@ return [ ], [ 'name' => '_APP_USAGE_AGGREGATION_INTERVAL', - 'description' => 'Deprecated since 1.0.0, use `_APP_USAGE_TIMESERIES_INTERVAL` and `_APP_USAGE_DATABASE_INTERVAL` instead.', - 'introduction' => '0.10.0', + 'description' => 'Interval value containing the number of seconds that the Appwrite usage process should wait before aggregating stats and syncing it to Database from TimeSeries data. The default value is 30 seconds. Reintroduced in 1.1.0.', + 'introduction' => '1.1.0', 'default' => '30', 'required' => false, 'question' => '', @@ -179,7 +179,7 @@ return [ ], [ 'name' => '_APP_USAGE_TIMESERIES_INTERVAL', - 'description' => 'Deprecated since _ use _APP_USAGE_AGGREGATION_INTERVAL instead.', + 'description' => 'Deprecated since 1.1.0 use _APP_USAGE_AGGREGATION_INTERVAL instead.', 'introduction' => '1.0.0', 'default' => '30', 'required' => false, @@ -188,7 +188,7 @@ return [ ], [ 'name' => '_APP_USAGE_DATABASE_INTERVAL', - 'description' => 'Deprecated since _ use _APP_USAGE_AGGREGATION_INTERVAL instead.', + 'description' => 'Deprecated since 1.1.0 use _APP_USAGE_AGGREGATION_INTERVAL instead.', 'introduction' => '1.0.0', 'default' => '900', 'required' => false, From 0d29968169fdc10be8c64a24abbbd0d31dbe8d5e Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 30 Oct 2022 05:14:46 +0000 Subject: [PATCH 58/69] refactor database listener callback --- app/controllers/shared/api.php | 115 +++++++++++++-------------------- 1 file changed, 44 insertions(+), 71 deletions(-) diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index c6597c339..915a31bc3 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -48,6 +48,47 @@ $parseLabel = function (string $label, array $responsePayload, array $requestPar return $label; }; +$databaseListener = function (string $event, Document $document, Stats $usage) { + $multiplier = 1; + if($event === Database::EVENT_DOCUMENT_DELETE) { + $multiplier = -1; + } + + $collection = $document->getCollection(); + switch ($collection) { + case 'users': + $usage->setParam('users.{scope}.count.total', 1 * $multiplier); + break; + case 'databases': + $usage->setParam('databases.{scope}.count.total', 1 * $multiplier); + break; + case 'buckets': + $usage->setParam('buckets.{scope}.count.total', 1 * $multiplier); + break; + case 'deployments': + $usage->setParam('deployments.{scope}.storage.size', $document->getAttribute('size') * $multiplier); + break; + default: + if (strpos($collection, 'bucket_') === 0) { + $usage + ->setParam('bucketId', $document->getAttribute('bucketId')) + ->setParam('files.{scope}.storage.size', $document->getAttribute('sizeOriginal') * $multiplier) + ->setParam('files.{scope}.count.total', 1 * $multiplier); + } elseif (strpos($collection, 'database_') === 0) { + $usage + ->setParam('databaseId', $document->getAttribute('databaseId')); + if (strpos($collection, '_collection_') != false) { + $usage + ->setParam('collectionId', $document->getAttribute('$collectionId')) + ->setParam('documents.{scope}.count.total', 1 * $multiplier); + } else { + $usage->setParam('collections.{scope}.count.total', 1 * $multiplier); + } + } + break; + } +}; + App::init() ->groups(['api']) ->inject('utopia') @@ -63,7 +104,7 @@ App::init() ->inject('database') ->inject('dbForProject') ->inject('mode') - ->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Event $events, Audit $audits, Mail $mails, Stats $usage, Delete $deletes, EventDatabase $database, Database $dbForProject, string $mode) { + ->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Event $events, Audit $audits, Mail $mails, Stats $usage, Delete $deletes, EventDatabase $database, Database $dbForProject, string $mode) use ($databaseListener) { $route = $utopia->match($request); @@ -160,77 +201,9 @@ App::init() $deletes->setProject($project); $database->setProject($project); - $dbForProject->on(Database::EVENT_DOCUMENT_CREATE, function ($event, Document $document) use ($usage) { - $collection = $document->getCollection(); - switch ($collection) { - case 'users': - $usage->setParam('users.{scope}.count.total', 1); - break; - case 'databases': - $usage->setParam('databases.{scope}.count.total', 1); - break; - case 'buckets': - $usage->setParam('buckets.{scope}.count.total', 1); - break; - case 'deployments': - $usage->setParam('deployments.{scope}.storage.size', $document->getAttribute('size')); - break; - default: - if (strpos($collection, 'bucket_') === 0) { - $usage - ->setParam('bucketId', $document->getAttribute('bucketId')) - ->setParam('files.{scope}.storage.size', $document->getAttribute('sizeOriginal')) - ->setParam('files.{scope}.count.total', 1); - } elseif (strpos($collection, 'database_') === 0) { - $usage - ->setParam('databaseId', $document->getAttribute('databaseId')); - if (strpos($collection, '_collection_') != false) { - $usage - ->setParam('collectionId', $document->getAttribute('$collectionId')) - ->setParam('documents.{scope}.count.total', 1); - } else { - $usage->setParam('collections.{scope}.count.total', 1); - } - } - break; - } - }); + $dbForProject->on(Database::EVENT_DOCUMENT_CREATE, fn ($event, Document $document) => $databaseListener($event, $document, $usage)); - $dbForProject->on(Database::EVENT_DOCUMENT_DELETE, function ($event, Document $document) use ($usage) { - $collection = $document->getCollection(); - switch ($collection) { - case 'users': - $usage->setParam('users.{scope}.count.total', -1); - break; - case 'databases': - $usage->setParam('databases.{scope}.count.total', -1); - break; - case 'buckets': - $usage->setParam('buckets.{scope}.count.total', -1); - break; - case 'deployments': - $usage->setParam('deployments.{scope}.storage.size', -$document->getAttribute('size')); - break; - default: - if (strpos($collection, 'bucket_') === 0) { - $usage - ->setParam('bucketId', $document->getAttribute('bucketId')) - ->setParam('files.{scope}.storage.size', -$document->getAttribute('sizeOriginal')) - ->setParam('files.{scope}.count.total', -1); - } elseif (strpos($collection, 'database_') === 0) { - $usage - ->setParam('databaseId', $document->getAttribute('databaseId')); - if (strpos($collection, '_collection_') != false) { - $usage - ->setParam('collectionId', $document->getAttribute('collectionId')) - ->setParam('documents.{scope}.count.total', -1); - } else { - $usage->setParam('collections.{scope}.count.total', -1); - } - } - break; - } - }); + $dbForProject->on(Database::EVENT_DOCUMENT_DELETE, fn ($event, Document $document) => $databaseListener($event, $document, $usage)); $useCache = $route->getLabel('cache', false); From 7433a260db6fcbc1c11adc0e828d33270b84a988 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 30 Oct 2022 05:16:51 +0000 Subject: [PATCH 59/69] fix formatting --- app/controllers/shared/api.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 915a31bc3..6aeda5abe 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -50,7 +50,7 @@ $parseLabel = function (string $label, array $responsePayload, array $requestPar $databaseListener = function (string $event, Document $document, Stats $usage) { $multiplier = 1; - if($event === Database::EVENT_DOCUMENT_DELETE) { + if ($event === Database::EVENT_DOCUMENT_DELETE) { $multiplier = -1; } From bd83da3331bd0671224e057ee63921e0ac811db5 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 31 Oct 2022 04:30:19 +0000 Subject: [PATCH 60/69] upgrade databases --- composer.json | 6 ++--- composer.lock | 65 +++++++++++++++++++++++---------------------------- 2 files changed, 32 insertions(+), 39 deletions(-) diff --git a/composer.json b/composer.json index 1e28c3965..6c3cf1950 100644 --- a/composer.json +++ b/composer.json @@ -45,13 +45,13 @@ "appwrite/php-runtimes": "0.11.*", "utopia-php/framework": "0.21.*", "utopia-php/logger": "0.3.*", - "utopia-php/abuse": "0.14.*", + "utopia-php/abuse": "0.15.*", "utopia-php/analytics": "0.2.*", - "utopia-php/audit": "0.15.*", + "utopia-php/audit": "0.16.*", "utopia-php/cache": "0.6.*", "utopia-php/cli": "0.13.*", "utopia-php/config": "0.2.*", - "utopia-php/database": "0.27.0 as 0.26.0", + "utopia-php/database": "0.27.0", "utopia-php/locale": "0.4.*", "utopia-php/registry": "0.5.*", "utopia-php/preloader": "0.2.*", diff --git a/composer.lock b/composer.lock index acf2058b3..da90bebf2 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "0e0aa8681829f5c1c68b5b80044f12a5", + "content-hash": "fe23114c0eec940a582137c2ecac8e6a", "packages": [ { "name": "adhocore/jwt", @@ -693,16 +693,16 @@ }, { "name": "guzzlehttp/psr7", - "version": "2.4.1", + "version": "2.4.3", "source": { "type": "git", "url": "https://github.com/guzzle/psr7.git", - "reference": "69568e4293f4fa993f3b0e51c9723e1e17c41379" + "reference": "67c26b443f348a51926030c83481b85718457d3d" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/guzzle/psr7/zipball/69568e4293f4fa993f3b0e51c9723e1e17c41379", - "reference": "69568e4293f4fa993f3b0e51c9723e1e17c41379", + "url": "https://api.github.com/repos/guzzle/psr7/zipball/67c26b443f348a51926030c83481b85718457d3d", + "reference": "67c26b443f348a51926030c83481b85718457d3d", "shasum": "" }, "require": { @@ -792,7 +792,7 @@ ], "support": { "issues": "https://github.com/guzzle/psr7/issues", - "source": "https://github.com/guzzle/psr7/tree/2.4.1" + "source": "https://github.com/guzzle/psr7/tree/2.4.3" }, "funding": [ { @@ -808,7 +808,7 @@ "type": "tidelift" } ], - "time": "2022-08-28T14:45:39+00:00" + "time": "2022-10-26T14:07:24+00:00" }, { "name": "influxdb/influxdb-php", @@ -1741,23 +1741,23 @@ }, { "name": "utopia-php/abuse", - "version": "0.14.0", + "version": "0.15.0", "source": { "type": "git", "url": "https://github.com/utopia-php/abuse.git", - "reference": "1a5da248e74c1bfc39bc440fa949de6935acceeb" + "reference": "fa7684d3162d370cf17df349066dd6535dd7264a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/abuse/zipball/1a5da248e74c1bfc39bc440fa949de6935acceeb", - "reference": "1a5da248e74c1bfc39bc440fa949de6935acceeb", + "url": "https://api.github.com/repos/utopia-php/abuse/zipball/fa7684d3162d370cf17df349066dd6535dd7264a", + "reference": "fa7684d3162d370cf17df349066dd6535dd7264a", "shasum": "" }, "require": { "ext-curl": "*", "ext-pdo": "*", "php": ">=8.0", - "utopia-php/database": "0.26.*" + "utopia-php/database": "0.27.*" }, "require-dev": { "phpunit/phpunit": "^9.4", @@ -1789,9 +1789,9 @@ ], "support": { "issues": "https://github.com/utopia-php/abuse/issues", - "source": "https://github.com/utopia-php/abuse/tree/0.14.0" + "source": "https://github.com/utopia-php/abuse/tree/0.15.0" }, - "time": "2022-10-14T11:26:39+00:00" + "time": "2022-10-30T08:11:12+00:00" }, { "name": "utopia-php/analytics", @@ -1850,22 +1850,22 @@ }, { "name": "utopia-php/audit", - "version": "0.15.0", + "version": "0.16.0", "source": { "type": "git", "url": "https://github.com/utopia-php/audit.git", - "reference": "937ffd13e7a5ac9ad220b329247569ef2a4881d9" + "reference": "fd8a0ea973f50443e6b7b8d0728d4c1e136da0ef" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/audit/zipball/937ffd13e7a5ac9ad220b329247569ef2a4881d9", - "reference": "937ffd13e7a5ac9ad220b329247569ef2a4881d9", + "url": "https://api.github.com/repos/utopia-php/audit/zipball/fd8a0ea973f50443e6b7b8d0728d4c1e136da0ef", + "reference": "fd8a0ea973f50443e6b7b8d0728d4c1e136da0ef", "shasum": "" }, "require": { "ext-pdo": "*", "php": ">=8.0", - "utopia-php/database": "0.26.*" + "utopia-php/database": "0.27.*" }, "require-dev": { "phpunit/phpunit": "^9.3", @@ -1891,9 +1891,9 @@ ], "support": { "issues": "https://github.com/utopia-php/audit/issues", - "source": "https://github.com/utopia-php/audit/tree/0.15.0" + "source": "https://github.com/utopia-php/audit/tree/0.16.0" }, - "time": "2022-10-14T11:39:18+00:00" + "time": "2022-10-30T08:11:26+00:00" }, { "name": "utopia-php/cache", @@ -3525,16 +3525,16 @@ }, { "name": "phpunit/php-code-coverage", - "version": "9.2.17", + "version": "9.2.18", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/php-code-coverage.git", - "reference": "aa94dc41e8661fe90c7316849907cba3007b10d8" + "reference": "12fddc491826940cf9b7e88ad9664cf51f0f6d0a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/aa94dc41e8661fe90c7316849907cba3007b10d8", - "reference": "aa94dc41e8661fe90c7316849907cba3007b10d8", + "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/12fddc491826940cf9b7e88ad9664cf51f0f6d0a", + "reference": "12fddc491826940cf9b7e88ad9664cf51f0f6d0a", "shasum": "" }, "require": { @@ -3590,7 +3590,7 @@ ], "support": { "issues": "https://github.com/sebastianbergmann/php-code-coverage/issues", - "source": "https://github.com/sebastianbergmann/php-code-coverage/tree/9.2.17" + "source": "https://github.com/sebastianbergmann/php-code-coverage/tree/9.2.18" }, "funding": [ { @@ -3598,7 +3598,7 @@ "type": "github" } ], - "time": "2022-08-30T12:24:04+00:00" + "time": "2022-10-27T13:35:33+00:00" }, { "name": "phpunit/php-file-iterator", @@ -5347,14 +5347,7 @@ "time": "2022-09-28T08:42:51+00:00" } ], - "aliases": [ - { - "package": "utopia-php/database", - "version": "0.27.0.0", - "alias": "0.26.0", - "alias_normalized": "0.26.0.0" - } - ], + "aliases": [], "minimum-stability": "stable", "stability-flags": [], "prefer-stable": false, @@ -5380,5 +5373,5 @@ "platform-overrides": { "php": "8.0" }, - "plugin-api-version": "2.3.0" + "plugin-api-version": "2.2.0" } From 67113971d9b6f539e1582db43cbf689a6453a54a Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 31 Oct 2022 07:27:43 +0000 Subject: [PATCH 61/69] fix model --- src/Appwrite/Utopia/Response/Model/Provider.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Utopia/Response/Model/Provider.php b/src/Appwrite/Utopia/Response/Model/Provider.php index 6bd533d9b..0f1499350 100644 --- a/src/Appwrite/Utopia/Response/Model/Provider.php +++ b/src/Appwrite/Utopia/Response/Model/Provider.php @@ -10,7 +10,7 @@ class Provider extends Model /** * @var bool */ - protected $public = false; + protected bool $public = false; public function __construct() { From cf5ecd5a62d13fda85ba00245a0256d39f0e7832 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 31 Oct 2022 07:27:51 +0000 Subject: [PATCH 62/69] ignore data without projectInternalId --- src/Appwrite/Usage/Calculators/TimeSeries.php | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Appwrite/Usage/Calculators/TimeSeries.php b/src/Appwrite/Usage/Calculators/TimeSeries.php index de8f3452d..47fba0318 100644 --- a/src/Appwrite/Usage/Calculators/TimeSeries.php +++ b/src/Appwrite/Usage/Calculators/TimeSeries.php @@ -491,7 +491,9 @@ class TimeSeries extends Calculator } $value = (!empty($point['value'])) ? $point['value'] : 0; - + if(empty($point['projectInternalId'] ?? null)) { + return; + } $this->createOrUpdateMetric( $point['projectInternalId'], $point['time'], From 4efabe93701b0c30da10cb0f3fd5705cdfa98b9d Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 31 Oct 2022 07:44:44 +0000 Subject: [PATCH 63/69] fix formatting --- src/Appwrite/Usage/Calculators/TimeSeries.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Usage/Calculators/TimeSeries.php b/src/Appwrite/Usage/Calculators/TimeSeries.php index 47fba0318..7b9497e66 100644 --- a/src/Appwrite/Usage/Calculators/TimeSeries.php +++ b/src/Appwrite/Usage/Calculators/TimeSeries.php @@ -491,7 +491,7 @@ class TimeSeries extends Calculator } $value = (!empty($point['value'])) ? $point['value'] : 0; - if(empty($point['projectInternalId'] ?? null)) { + if (empty($point['projectInternalId'] ?? null)) { return; } $this->createOrUpdateMetric( From 7989518eecb033ad36b217c0668f62378bdc139e Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 31 Oct 2022 09:07:06 +0000 Subject: [PATCH 64/69] buildTime attribute to deployment --- app/config/collections.php | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/app/config/collections.php b/app/config/collections.php index 6f5551630..401a4532f 100644 --- a/app/config/collections.php +++ b/app/config/collections.php @@ -2327,6 +2327,17 @@ $collections = [ 'array' => false, 'filters' => [], ], + [ + '$id' => ID::custom('buildTime'), + 'type' => Database::VAR_INTEGER, + 'format' => '', + 'size' => 0, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], [ 'array' => false, '$id' => ID::custom('entrypoint'), From 24a140c3af3bfccf63d85acb429302ff09c62bf4 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 31 Oct 2022 10:17:08 +0000 Subject: [PATCH 65/69] assert buildTime is greater than 0 --- tests/e2e/Services/Functions/FunctionsCustomServerTest.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php index 7694bf8ac..7e12d9dc3 100644 --- a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php +++ b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php @@ -569,7 +569,7 @@ class FunctionsCustomServerTest extends Scope ], $this->getHeaders())); $this->assertEquals(200, $function['headers']['status-code']); - $this->assertEquals(0, $function['body']['buildTime']); + $this->assertGreaterThan(0, $function['body']['buildTime']); /** * Test for FAILURE From 4d3ca5a06439eb253aa11a6c6cc1243f55cf6504 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 1 Nov 2022 11:41:52 +0545 Subject: [PATCH 66/69] revert back function test --- tests/e2e/Services/Functions/FunctionsCustomServerTest.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php index 7e12d9dc3..7694bf8ac 100644 --- a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php +++ b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php @@ -569,7 +569,7 @@ class FunctionsCustomServerTest extends Scope ], $this->getHeaders())); $this->assertEquals(200, $function['headers']['status-code']); - $this->assertGreaterThan(0, $function['body']['buildTime']); + $this->assertEquals(0, $function['body']['buildTime']); /** * Test for FAILURE From 41d31c28d0ac05917a4e64a8c055525e5814d051 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Wed, 2 Nov 2022 15:38:59 +0545 Subject: [PATCH 67/69] remove buildTime from deployments collection --- app/config/collections.php | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/app/config/collections.php b/app/config/collections.php index 89ecffd44..06cb785f5 100644 --- a/app/config/collections.php +++ b/app/config/collections.php @@ -2327,17 +2327,6 @@ $collections = [ 'array' => false, 'filters' => [], ], - [ - '$id' => ID::custom('buildTime'), - 'type' => Database::VAR_INTEGER, - 'format' => '', - 'size' => 0, - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ], [ 'array' => false, '$id' => ID::custom('entrypoint'), From 4b5d537eeeee45db9d25cf8a3c508f3f108affff Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Wed, 2 Nov 2022 15:41:50 +0545 Subject: [PATCH 68/69] Apply suggestions from code review Co-authored-by: Torsten Dittmann --- app/controllers/shared/api.php | 2 +- src/Appwrite/Usage/Stats.php | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 6aeda5abe..261f709d4 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -77,7 +77,7 @@ $databaseListener = function (string $event, Document $document, Stats $usage) { } elseif (strpos($collection, 'database_') === 0) { $usage ->setParam('databaseId', $document->getAttribute('databaseId')); - if (strpos($collection, '_collection_') != false) { + if (strpos($collection, '_collection_') !== false) { $usage ->setParam('collectionId', $document->getAttribute('$collectionId')) ->setParam('documents.{scope}.count.total', 1 * $multiplier); diff --git a/src/Appwrite/Usage/Stats.php b/src/Appwrite/Usage/Stats.php index 510584c87..e6e005666 100644 --- a/src/Appwrite/Usage/Stats.php +++ b/src/Appwrite/Usage/Stats.php @@ -110,7 +110,7 @@ class Stats foreach ($usersMetrics as $metric) { $value = $this->params[$metric] ?? 0; - if ($value == 1 || $value == -1) { + if ($value === 1 || $value === -1) { $this->statsd->count($metric . $tags, $value); } } @@ -135,7 +135,7 @@ class Stats foreach ($dbMetrics as $metric) { $value = $this->params[$metric] ?? 0; - if ($value == 1 || $value == -1) { + if ($value === 1 || $value === -1) { $dbTags = $tags . ",collectionId=" . ($this->params['collectionId'] ?? '') . ",databaseId=" . ($this->params['databaseId'] ?? ''); $this->statsd->count($metric . $dbTags, $value); } @@ -157,7 +157,7 @@ class Stats foreach ($storageMertics as $metric) { $value = $this->params[$metric] ?? 0; - if ($value >= 1 || $value == -1 || $value < -1) { + if ($value !== 0) { $storageTags = $tags . ",bucketId=" . ($this->params['bucketId'] ?? ''); $this->statsd->count($metric . $storageTags, $value); } @@ -194,7 +194,7 @@ class Stats $this->statsd->count('project.{scope}.storage.size' . $tags, $deploymentSize + $storageSize); } - if ($deploymentSize > 0 || $deploymentSize <= -1) { + if ($deploymentSize !== 0) { $this->statsd->count('deployments.{scope}.storage.size' . $functionTags, $deploymentSize); } From 4479497dfef8b1b17cd567ab0e8a59481bc2004d Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 10 Nov 2022 05:28:56 +0000 Subject: [PATCH 69/69] fix difference in naming --- src/Appwrite/Usage/Stats.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Appwrite/Usage/Stats.php b/src/Appwrite/Usage/Stats.php index 00298238f..3da786492 100644 --- a/src/Appwrite/Usage/Stats.php +++ b/src/Appwrite/Usage/Stats.php @@ -91,8 +91,8 @@ class Stats $this->statsd->increment('project.{scope}.network.requests' . $tags . ',method=' . \strtolower($httpMethod)); } - $inbound = $this->params['networkRequestSize'] ?? 0; - $outbound = $this->params['networkResponseSize'] ?? 0; + $inbound = $this->params['project.{scope}.network.inbound'] ?? 0; + $outbound = $this->params['project.{scope}.network.outbound'] ?? 0; $this->statsd->count('project.{scope}.network.inbound' . $tags, $inbound); $this->statsd->count('project.{scope}.network.outbound' . $tags, $outbound); $this->statsd->count('project.{scope}.network.bandwidth' . $tags, $inbound + $outbound);