CodeMirror/appwrite
1
0
Fork 0
mirror of synced 2024-07-06 23:21:05 +12:00
Code Issues Projects Releases Wiki Activity

Merge pull request #4582 from appwrite/revert-4545-feat-db-pools-usage-fix

Browse source 
Revert "Fix usage on DB Pools"
This commit is contained in:
Christy Jacob 2022-10-26 21:36:03 +05:30 committed by GitHub
parent 95b6d5d428 c2be54ca7f
commit 286e041c7f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 126 additions and 156 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

24
app/cli.php
View file

@ -12,7 +12,6 @@ use Utopia\Config\Config;
use Utopia\Database\Database; use Utopia\Database\Database;
use Utopia\Database\Validator\Authorization; use Utopia\Database\Validator\Authorization;
use InfluxDB\Database as InfluxDatabase; use InfluxDB\Database as InfluxDatabase;
use Utopia\Database\Document;
function getInfluxDB(): InfluxDatabase function getInfluxDB(): InfluxDatabase
{ {
@ -60,29 +59,6 @@ function getConsoleDB(): Database
return $database; return $database;
} }
function getProjectDB(Document $project): Database
{
global $register;
$pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */
if ($project->isEmpty() || $project->getId() === 'console') {
return getConsoleDB();
}
$dbAdapter = $pools
->get($project->getAttribute('database'))
->pop()
->getResource()
;
$database = new Database($dbAdapter, getCache());
$database->setNamespace('_' . $project->getInternalId());
return $database;
}
function getCache(): Cache function getCache(): Cache
{ {
global $register; global $register;

20
app/tasks/usage.php
View file

@ -9,10 +9,8 @@ use InfluxDB\Database as InfluxDatabase;
use Utopia\App; use Utopia\App;
use Utopia\CLI\Console; use Utopia\CLI\Console;
use Utopia\Database\Database as UtopiaDatabase; use Utopia\Database\Database as UtopiaDatabase;
use Utopia\Database\Document;
use Utopia\Database\Validator\Authorization; use Utopia\Database\Validator\Authorization;
use Utopia\Logger\Log; use Utopia\Logger\Log;
use Utopia\Registry\Registry;
use Utopia\Validator\WhiteList; use Utopia\Validator\WhiteList;
Authorization::disable(); Authorization::disable();
@ -52,10 +50,10 @@ $logError = function (Throwable $error, string $action = 'syncUsageStats') use (
Console::warning($error->getTraceAsString()); Console::warning($error->getTraceAsString());
}; };
function aggregateTimeseries(UtopiaDatabase $database, InfluxDatabase $influxDB, callable $getProjectDB, callable $logError): void function aggregateTimeseries(UtopiaDatabase $database, InfluxDatabase $influxDB, callable $logError): void
{ {
$interval = (int) App::getEnv('_APP_USAGE_TIMESERIES_INTERVAL', '30'); // 30 seconds (by default) $interval = (int) App::getEnv('_APP_USAGE_TIMESERIES_INTERVAL', '30'); // 30 seconds (by default)
$usage = new TimeSeries($database, $influxDB, $getProjectDB, $logError); $usage = new TimeSeries($database, $influxDB, $logError);
Console::loop(function () use ($interval, $usage) { Console::loop(function () use ($interval, $usage) {
$now = date('d-m-Y H:i:s', time()); $now = date('d-m-Y H:i:s', time());
@ -70,11 +68,11 @@ function aggregateTimeseries(UtopiaDatabase $database, InfluxDatabase $influxDB,
}, $interval); }, $interval);
} }
function aggregateDatabase(UtopiaDatabase $database, callable $getProjectDB, Registry $register, callable $logError): void function aggregateDatabase(UtopiaDatabase $database, callable $logError): void
{ {
$interval = (int) App::getEnv('_APP_USAGE_DATABASE_INTERVAL', '900'); // 15 minutes (by default) $interval = (int) App::getEnv('_APP_USAGE_DATABASE_INTERVAL', '900'); // 15 minutes (by default)
$usage = new Database($database, $getProjectDB, $register, $logError); $usage = new Database($database, $logError);
$aggregrator = new Aggregator($database, $getProjectDB, $register, $logError); $aggregrator = new Aggregator($database, $logError);
Console::loop(function () use ($interval, $usage, $aggregrator) { Console::loop(function () use ($interval, $usage, $aggregrator) {
$now = date('d-m-Y H:i:s', time()); $now = date('d-m-Y H:i:s', time());
@ -93,23 +91,21 @@ $cli
->task('usage') ->task('usage')
->param('type', 'timeseries', new WhiteList(['timeseries', 'database'])) ->param('type', 'timeseries', new WhiteList(['timeseries', 'database']))
->desc('Schedules syncing data from influxdb to Appwrite console db') ->desc('Schedules syncing data from influxdb to Appwrite console db')
->action(function (string $type) use ($logError, $register) { ->action(function (string $type) use ($logError) {
Console::title('Usage Aggregation V1'); Console::title('Usage Aggregation V1');
Console::success(APP_NAME . ' usage aggregation process v1 has started'); Console::success(APP_NAME . ' usage aggregation process v1 has started');
$database = getConsoleDB(); $database = getConsoleDB();
$influxDB = getInfluxDB(); $influxDB = getInfluxDB();
$getProjectDB = fn (Document $project) => getProjectDB($project);
switch ($type) { switch ($type) {
case 'timeseries': case 'timeseries':
aggregateTimeseries($database, $influxDB, $getProjectDB, $logError); aggregateTimeseries($database, $influxDB, $logError);
break; break;
case 'database': case 'database':
aggregateDatabase($database, $getProjectDB, $register, $logError); aggregateDatabase($database, $logError);
break; break;
default: default:
Console::error("Unsupported usage aggregation type"); Console::error("Unsupported usage aggregation type");
} }
$register->get('pools')->reclaim();
}); });

108
src/Appwrite/Usage/Calculators/Aggregator.php
View file

@ -9,8 +9,10 @@ use Utopia\Database\Query;
class Aggregator extends Database class Aggregator extends Database
{ {
protected function aggregateDatabaseMetrics(UtopiaDatabase $database, Document $project): void protected function aggregateDatabaseMetrics(string $projectId): void
{ {
$this->database->setNamespace('_' . $projectId);
$databasesGeneralMetrics = [ $databasesGeneralMetrics = [
'databases.$all.requests.create', 'databases.$all.requests.create',
'databases.$all.requests.read', 'databases.$all.requests.read',
@ -27,8 +29,8 @@ class Aggregator extends Database
]; ];
foreach ($databasesGeneralMetrics as $metric) { foreach ($databasesGeneralMetrics as $metric) {
$this->aggregateDailyMetric($database, $project, $metric); $this->aggregateDailyMetric($projectId, $metric);
$this->aggregateMonthlyMetric($database, $project, $metric); $this->aggregateMonthlyMetric($projectId, $metric);
} }
$databasesDatabaseMetrics = [ $databasesDatabaseMetrics = [
@ -42,12 +44,12 @@ class Aggregator extends Database
'documents.databaseId.requests.delete', 'documents.databaseId.requests.delete',
]; ];
$this->foreachDocument($project, 'databases', [], function (Document $db) use ($databasesDatabaseMetrics, $project, $database) { $this->foreachDocument($projectId, 'databases', [], function (Document $database) use ($databasesDatabaseMetrics, $projectId) {
$databaseId = $db->getId(); $databaseId = $database->getId();
foreach ($databasesDatabaseMetrics as $metric) { foreach ($databasesDatabaseMetrics as $metric) {
$metric = str_replace('databaseId', $databaseId, $metric); $metric = str_replace('databaseId', $databaseId, $metric);
$this->aggregateDailyMetric($database, $project, $metric); $this->aggregateDailyMetric($projectId, $metric);
$this->aggregateMonthlyMetric($database, $project, $metric); $this->aggregateMonthlyMetric($projectId, $metric);
} }
$databasesCollectionMetrics = [ $databasesCollectionMetrics = [
@ -57,19 +59,21 @@ class Aggregator extends Database
'documents.' . $databaseId . '/collectionId.requests.delete', 'documents.' . $databaseId . '/collectionId.requests.delete',
]; ];
$this->foreachDocument($project, 'database_' . $db->getInternalId(), [], function (Document $collection) use ($databasesCollectionMetrics, $project, $database) { $this->foreachDocument($projectId, 'database_' . $database->getInternalId(), [], function (Document $collection) use ($databasesCollectionMetrics, $projectId) {
$collectionId = $collection->getId(); $collectionId = $collection->getId();
foreach ($databasesCollectionMetrics as $metric) { foreach ($databasesCollectionMetrics as $metric) {
$metric = str_replace('collectionId', $collectionId, $metric); $metric = str_replace('collectionId', $collectionId, $metric);
$this->aggregateDailyMetric($database, $project, $metric); $this->aggregateDailyMetric($projectId, $metric);
$this->aggregateMonthlyMetric($database, $project, $metric); $this->aggregateMonthlyMetric($projectId, $metric);
} }
}); });
}); });
} }
protected function aggregateStorageMetrics(UtopiaDatabase $database, Document $project): void protected function aggregateStorageMetrics(string $projectId): void
{ {
$this->database->setNamespace('_' . $projectId);
$storageGeneralMetrics = [ $storageGeneralMetrics = [
'buckets.$all.requests.create', 'buckets.$all.requests.create',
'buckets.$all.requests.read', 'buckets.$all.requests.read',
@ -82,8 +86,8 @@ class Aggregator extends Database
]; ];
foreach ($storageGeneralMetrics as $metric) { foreach ($storageGeneralMetrics as $metric) {
$this->aggregateDailyMetric($database, $project, $metric); $this->aggregateDailyMetric($projectId, $metric);
$this->aggregateMonthlyMetric($database, $project, $metric); $this->aggregateMonthlyMetric($projectId, $metric);
} }
$storageBucketMetrics = [ $storageBucketMetrics = [
@ -93,18 +97,20 @@ class Aggregator extends Database
'files.bucketId.requests.delete', 'files.bucketId.requests.delete',
]; ];
$this->foreachDocument($project, 'buckets', [], function (Document $bucket) use ($storageBucketMetrics, $project, $database) { $this->foreachDocument($projectId, 'buckets', [], function (Document $bucket) use ($storageBucketMetrics, $projectId) {
$bucketId = $bucket->getId(); $bucketId = $bucket->getId();
foreach ($storageBucketMetrics as $metric) { foreach ($storageBucketMetrics as $metric) {
$metric = str_replace('bucketId', $bucketId, $metric); $metric = str_replace('bucketId', $bucketId, $metric);
$this->aggregateDailyMetric($database, $project, $metric); $this->aggregateDailyMetric($projectId, $metric);
$this->aggregateMonthlyMetric($database, $project, $metric); $this->aggregateMonthlyMetric($projectId, $metric);
} }
}); });
} }
protected function aggregateFunctionMetrics(UtopiaDatabase $database, Document $project): void protected function aggregateFunctionMetrics(string $projectId): void
{ {
$this->database->setNamespace('_' . $projectId);
$functionsGeneralMetrics = [ $functionsGeneralMetrics = [
'project.$all.compute.total', 'project.$all.compute.total',
'project.$all.compute.time', 'project.$all.compute.time',
@ -119,8 +125,8 @@ class Aggregator extends Database
]; ];
foreach ($functionsGeneralMetrics as $metric) { foreach ($functionsGeneralMetrics as $metric) {
$this->aggregateDailyMetric($database, $project, $metric); $this->aggregateDailyMetric($projectId, $metric);
$this->aggregateMonthlyMetric($database, $project, $metric); $this->aggregateMonthlyMetric($projectId, $metric);
} }
$functionMetrics = [ $functionMetrics = [
@ -134,17 +140,17 @@ class Aggregator extends Database
'builds.functionId.compute.time', 'builds.functionId.compute.time',
]; ];
$this->foreachDocument($project, 'functions', [], function (Document $function) use ($functionMetrics, $project, $database) { $this->foreachDocument($projectId, 'functions', [], function (Document $function) use ($functionMetrics, $projectId) {
$functionId = $function->getId(); $functionId = $function->getId();
foreach ($functionMetrics as $metric) { foreach ($functionMetrics as $metric) {
$metric = str_replace('functionId', $functionId, $metric); $metric = str_replace('functionId', $functionId, $metric);
$this->aggregateDailyMetric($database, $project, $metric); $this->aggregateDailyMetric($projectId, $metric);
$this->aggregateMonthlyMetric($database, $project, $metric); $this->aggregateMonthlyMetric($projectId, $metric);
} }
}); });
} }
protected function aggregateUsersMetrics(UtopiaDatabase $database, Document $project): void protected function aggregateUsersMetrics(string $projectId): void
{ {
$metrics = [ $metrics = [
'users.$all.requests.create', 'users.$all.requests.create',
@ -156,50 +162,50 @@ class Aggregator extends Database
]; ];
foreach ($metrics as $metric) { foreach ($metrics as $metric) {
$this->aggregateDailyMetric($database, $project, $metric); $this->aggregateDailyMetric($projectId, $metric);
$this->aggregateMonthlyMetric($database, $project, $metric); $this->aggregateMonthlyMetric($projectId, $metric);
} }
} }
protected function aggregateGeneralMetrics(UtopiaDatabase $database, Document $project): void protected function aggregateGeneralMetrics(string $projectId): void
{ {
$this->aggregateDailyMetric($database, $project, 'project.$all.network.requests'); $this->aggregateDailyMetric($projectId, 'project.$all.network.requests');
$this->aggregateDailyMetric($database, $project, 'project.$all.network.bandwidth'); $this->aggregateDailyMetric($projectId, 'project.$all.network.bandwidth');
$this->aggregateDailyMetric($database, $project, 'project.$all.network.inbound'); $this->aggregateDailyMetric($projectId, 'project.$all.network.inbound');
$this->aggregateDailyMetric($database, $project, 'project.$all.network.outbound'); $this->aggregateDailyMetric($projectId, 'project.$all.network.outbound');
$this->aggregateMonthlyMetric($database, $project, 'project.$all.network.requests'); $this->aggregateMonthlyMetric($projectId, 'project.$all.network.requests');
$this->aggregateMonthlyMetric($database, $project, 'project.$all.network.bandwidth'); $this->aggregateMonthlyMetric($projectId, 'project.$all.network.bandwidth');
$this->aggregateMonthlyMetric($database, $project, 'project.$all.network.inbound'); $this->aggregateMonthlyMetric($projectId, 'project.$all.network.inbound');
$this->aggregateMonthlyMetric($database, $project, 'project.$all.network.outbound'); $this->aggregateMonthlyMetric($projectId, 'project.$all.network.outbound');
} }
protected function aggregateDailyMetric(UtopiaDatabase $database, Document $project, string $metric): void protected function aggregateDailyMetric(string $projectId, string $metric): void
{ {
$beginOfDay = DateTime::createFromFormat('Y-m-d\TH:i:s.v', \date('Y-m-d\T00:00:00.000'))->format(DateTime::RFC3339); $beginOfDay = DateTime::createFromFormat('Y-m-d\TH:i:s.v', \date('Y-m-d\T00:00:00.000'))->format(DateTime::RFC3339);
$endOfDay = DateTime::createFromFormat('Y-m-d\TH:i:s.v', \date('Y-m-d\T23:59:59.999'))->format(DateTime::RFC3339); $endOfDay = DateTime::createFromFormat('Y-m-d\TH:i:s.v', \date('Y-m-d\T23:59:59.999'))->format(DateTime::RFC3339);
$database = call_user_func($this->getProjectDB, $project); $this->database->setNamespace('_' . $projectId);
$value = (int) $database->sum('stats', 'value', [ $value = (int) $this->database->sum('stats', 'value', [
Query::equal('metric', [$metric]), Query::equal('metric', [$metric]),
Query::equal('period', ['30m']), Query::equal('period', ['30m']),
Query::greaterThanEqual('time', $beginOfDay), Query::greaterThanEqual('time', $beginOfDay),
Query::lessThanEqual('time', $endOfDay), Query::lessThanEqual('time', $endOfDay),
]); ]);
$this->createOrUpdateMetric($database, $project->getId(), $metric, '1d', $beginOfDay, $value); $this->createOrUpdateMetric($projectId, $metric, '1d', $beginOfDay, $value);
} }
protected function aggregateMonthlyMetric(UtopiaDatabase $database, Document $project, string $metric): void protected function aggregateMonthlyMetric(string $projectId, string $metric): void
{ {
$beginOfMonth = DateTime::createFromFormat('Y-m-d\TH:i:s.v', \date('Y-m-01\T00:00:00.000'))->format(DateTime::RFC3339); $beginOfMonth = DateTime::createFromFormat('Y-m-d\TH:i:s.v', \date('Y-m-01\T00:00:00.000'))->format(DateTime::RFC3339);
$endOfMonth = DateTime::createFromFormat('Y-m-d\TH:i:s.v', \date('Y-m-t\T23:59:59.999'))->format(DateTime::RFC3339); $endOfMonth = DateTime::createFromFormat('Y-m-d\TH:i:s.v', \date('Y-m-t\T23:59:59.999'))->format(DateTime::RFC3339);
$database = call_user_func($this->getProjectDB, $project); $this->database->setNamespace('_' . $projectId);
$value = (int) $database->sum('stats', 'value', [ $value = (int) $this->database->sum('stats', 'value', [
Query::equal('metric', [$metric]), Query::equal('metric', [$metric]),
Query::equal('period', ['1d']), Query::equal('period', ['1d']),
Query::greaterThanEqual('time', $beginOfMonth), Query::greaterThanEqual('time', $beginOfMonth),
Query::lessThanEqual('time', $endOfMonth), Query::lessThanEqual('time', $endOfMonth),
]); ]);
$this->createOrUpdateMetric($database, $project->getId(), $metric, '1mo', $beginOfMonth, $value); $this->createOrUpdateMetric($projectId, $metric, '1mo', $beginOfMonth, $value);
} }
/** /**
@ -210,14 +216,16 @@ class Aggregator extends Database
*/ */
public function collect(): void public function collect(): void
{ {
$this->foreachDocument(new Document(['$id' => 'console']), 'projects', [], function (Document $project) { $this->foreachDocument('console', 'projects', [], function (Document $project) {
$database = call_user_func($this->getProjectDB, $project); $projectId = $project->getInternalId();
$this->aggregateGeneralMetrics($database, $project);
$this->aggregateFunctionMetrics($database, $project); // Aggregate new metrics from already collected usage metrics
$this->aggregateDatabaseMetrics($database, $project); // for lower time period (1day and 1 month metric from 30 minute metrics)
$this->aggregateStorageMetrics($database, $project); $this->aggregateGeneralMetrics($projectId);
$this->aggregateUsersMetrics($database, $project); $this->aggregateFunctionMetrics($projectId);
$this->register->get('pools')->reclaim(); $this->aggregateDatabaseMetrics($projectId);
$this->aggregateStorageMetrics($projectId);
$this->aggregateUsersMetrics($projectId);
}); });
} }
} }

116
src/Appwrite/Usage/Calculators/Database.php
View file

@ -10,11 +10,9 @@ use Utopia\Database\Document;
use Utopia\Database\Exception\Authorization; use Utopia\Database\Exception\Authorization;
use Utopia\Database\Exception\Structure; use Utopia\Database\Exception\Structure;
use Utopia\Database\Query; use Utopia\Database\Query;
use Utopia\Registry\Registry;
class Database extends Calculator class Database extends Calculator
{ {
protected Registry $register;
protected array $periods = [ protected array $periods = [
[ [
'key' => '30m', 'key' => '30m',
@ -26,11 +24,9 @@ class Database extends Calculator
], ],
]; ];
public function __construct(UtopiaDatabase $database, callable $getProjectDB, Registry $register, callable $errorHandler = null) public function __construct(UtopiaDatabase $database, callable $errorHandler = null)
{ {
$this->register = $register;
$this->database = $database; $this->database = $database;
$this->getProjectDB = $getProjectDB;
$this->errorHandler = $errorHandler; $this->errorHandler = $errorHandler;
} }
@ -39,8 +35,7 @@ class Database extends Calculator
* *
* Create given metric for each defined period * Create given metric for each defined period
* *
* @param UtopiaDatabase $database * @param string $projectId
* @param Document $project
* @param string $metric * @param string $metric
* @param int $value * @param int $value
* @param bool $monthly * @param bool $monthly
@ -48,7 +43,7 @@ class Database extends Calculator
* @throws Authorization * @throws Authorization
* @throws Structure * @throws Structure
*/ */
protected function createPerPeriodMetric(UtopiaDatabase $database, string $projectId, string $metric, int $value, bool $monthly = false): void protected function createPerPeriodMetric(string $projectId, string $metric, int $value, bool $monthly = false): void
{ {
foreach ($this->periods as $options) { foreach ($this->periods as $options) {
$period = $options['key']; $period = $options['key'];
@ -61,13 +56,13 @@ class Database extends Calculator
} else { } else {
throw new Exception("Period type not found", 500); throw new Exception("Period type not found", 500);
} }
$this->createOrUpdateMetric($database, $projectId, $metric, $period, $time, $value); $this->createOrUpdateMetric($projectId, $metric, $period, $time, $value);
} }
// Required for billing // Required for billing
if ($monthly) { if ($monthly) {
$time = DateTime::createFromFormat('Y-m-d\TH:i:s.v', \date('Y-m-01\T00:00:00.000'))->format(DateTime::RFC3339); $time = DateTime::createFromFormat('Y-m-d\TH:i:s.v', \date('Y-m-01\T00:00:00.000'))->format(DateTime::RFC3339);
$this->createOrUpdateMetric($database, $projectId, $metric, '1mo', $time, $value); $this->createOrUpdateMetric($projectId, $metric, '1mo', $time, $value);
} }
} }
@ -76,8 +71,7 @@ class Database extends Calculator
* *
* Create or update each metric in the stats collection for the given project * Create or update each metric in the stats collection for the given project
* *
* @param UtopiaDatabase $database * @param string $projectId
* @param String $projectId
* @param string $metric * @param string $metric
* @param string $period * @param string $period
* @param string $time * @param string $time
@ -87,14 +81,15 @@ class Database extends Calculator
* @throws Authorization * @throws Authorization
* @throws Structure * @throws Structure
*/ */
protected function createOrUpdateMetric(UtopiaDatabase $database, string $projectId, string $metric, string $period, string $time, int $value): void protected function createOrUpdateMetric(string $projectId, string $metric, string $period, string $time, int $value): void
{ {
$id = \md5("{$time}_{$period}_{$metric}"); $id = \md5("{$time}_{$period}_{$metric}");
$this->database->setNamespace('_' . $projectId);
try { try {
$document = $database->getDocument('stats', $id); $document = $this->database->getDocument('stats', $id);
if ($document->isEmpty()) { if ($document->isEmpty()) {
$database->createDocument('stats', new Document([ $this->database->createDocument('stats', new Document([
'$id' => $id, '$id' => $id,
'period' => $period, 'period' => $period,
'time' => $time, 'time' => $time,
@ -103,7 +98,7 @@ class Database extends Calculator
'type' => 2, // these are cumulative metrics 'type' => 2, // these are cumulative metrics
])); ]));
} else { } else {
$database->updateDocument( $this->database->updateDocument(
'stats', 'stats',
$document->getId(), $document->getId(),
$document->setAttribute('value', $value) $document->setAttribute('value', $value)
@ -123,7 +118,7 @@ class Database extends Calculator
* *
* Call provided callback for each document in the collection * Call provided callback for each document in the collection
* *
* @param Document $project * @param string $projectId
* @param string $collection * @param string $collection
* @param array $queries * @param array $queries
* @param callable $callback * @param callable $callback
@ -131,13 +126,13 @@ class Database extends Calculator
* @return void * @return void
* @throws Exception * @throws Exception
*/ */
protected function foreachDocument(Document $project, string $collection, array $queries, callable $callback): void protected function foreachDocument(string $projectId, string $collection, array $queries, callable $callback): void
{ {
$limit = 50; $limit = 50;
$results = []; $results = [];
$sum = $limit; $sum = $limit;
$latestDocument = null; $latestDocument = null;
$database = $project->getId() == 'console' ? $this->database : call_user_func($this->getProjectDB, $project); $this->database->setNamespace('_' . $projectId);
while ($sum === $limit) { while ($sum === $limit) {
try { try {
@ -148,7 +143,7 @@ class Database extends Calculator
$results = $this->database->find($collection, \array_merge($paginationQueries, $queries)); $results = $this->database->find($collection, \array_merge($paginationQueries, $queries));
} catch (\Exception $e) { } catch (\Exception $e) {
if (is_callable($this->errorHandler)) { if (is_callable($this->errorHandler)) {
call_user_func($this->errorHandler, $e, "fetch_documents_project_{$project->getId()}_collection_{$collection}"); call_user_func($this->errorHandler, $e, "fetch_documents_project_{$projectId}_collection_{$collection}");
return; return;
} else { } else {
throw $e; throw $e;
@ -174,7 +169,6 @@ class Database extends Calculator
* *
* Calculate sum of an attribute of documents in collection * Calculate sum of an attribute of documents in collection
* *
* @param UtopiaDatabase $database
* @param string $projectId * @param string $projectId
* @param string $collection * @param string $collection
* @param string $attribute * @param string $attribute
@ -183,15 +177,16 @@ class Database extends Calculator
* @return int * @return int
* @throws Exception * @throws Exception
*/ */
private function sum(UtopiaDatabase $database, string $projectId, string $collection, string $attribute, string $metric = null, int $multiplier = 1): int private function sum(string $projectId, string $collection, string $attribute, string $metric = null, int $multiplier = 1): int
{ {
$this->database->setNamespace('_' . $projectId);
try { try {
$sum = $database->sum($collection, $attribute); $sum = $this->database->sum($collection, $attribute);
$sum = (int) ($sum * $multiplier); $sum = (int) ($sum * $multiplier);
if (!is_null($metric)) { if (!is_null($metric)) {
$this->createPerPeriodMetric($database, $projectId, $metric, $sum); $this->createPerPeriodMetric($projectId, $metric, $sum);
} }
return $sum; return $sum;
} catch (Exception $e) { } catch (Exception $e) {
@ -209,7 +204,6 @@ class Database extends Calculator
* *
* Count number of documents in collection * Count number of documents in collection
* *
* @param UtopiaDatabase $database
* @param string $projectId * @param string $projectId
* @param string $collection * @param string $collection
* @param ?string $metric * @param ?string $metric
@ -217,12 +211,14 @@ class Database extends Calculator
* @return int * @return int
* @throws Exception * @throws Exception
*/ */
private function count(UtopiaDatabase $database, string $projectId, string $collection, ?string $metric = null): int private function count(string $projectId, string $collection, ?string $metric = null): int
{ {
$this->database->setNamespace('_' . $projectId);
try { try {
$count = $database->count($collection); $count = $this->database->count($collection);
if (!is_null($metric)) { if (!is_null($metric)) {
$this->createPerPeriodMetric($database, $projectId, (string) $metric, $count); $this->createPerPeriodMetric($projectId, (string) $metric, $count);
} }
return $count; return $count;
} catch (Exception $e) { } catch (Exception $e) {
@ -240,15 +236,14 @@ class Database extends Calculator
* *
* Total sum of storage used by deployments * Total sum of storage used by deployments
* *
* @param UtopiaDatabase $database
* @param string $projectId * @param string $projectId
* *
* @return int * @return int
* @throws Exception * @throws Exception
*/ */
private function deploymentsTotal(UtopiaDatabase $database, string $projectId): int private function deploymentsTotal(string $projectId): int
{ {
return $this->sum($database, $projectId, 'deployments', 'size', 'deployments.$all.storage.size'); return $this->sum($projectId, 'deployments', 'size', 'deployments.$all.storage.size');
} }
/** /**
@ -256,15 +251,14 @@ class Database extends Calculator
* *
* Metric: users.count * Metric: users.count
* *
* @param UtopiaDatabase $database
* @param string $projectId * @param string $projectId
* *
* @return void * @return void
* @throws Exception * @throws Exception
*/ */
private function usersStats(UtopiaDatabase $database, string $projectId): void private function usersStats(string $projectId): void
{ {
$this->count($database, $projectId, 'users', 'users.$all.count.total'); $this->count($projectId, 'users', 'users.$all.count.total');
} }
/** /**
@ -273,36 +267,35 @@ class Database extends Calculator
* Metrics: buckets.$all.count.total, files.$all.count.total, files.bucketId,count.total, * Metrics: buckets.$all.count.total, files.$all.count.total, files.bucketId,count.total,
* files.$all.storage.size, files.bucketId.storage.size, project.$all.storage.size * files.$all.storage.size, files.bucketId.storage.size, project.$all.storage.size
* *
* @param UtopiaDatabase $database * @param string $projectId
* @param Document $project
* *
* @return void * @return void
* @throws Authorization * @throws Authorization
* @throws Structure * @throws Structure
*/ */
private function storageStats(UtopiaDatabase $database, Document $project): void private function storageStats(string $projectId): void
{ {
$projectFilesTotal = 0; $projectFilesTotal = 0;
$projectFilesCount = 0; $projectFilesCount = 0;
$metric = 'buckets.$all.count.total'; $metric = 'buckets.$all.count.total';
$this->count($database, $project->getId(), 'buckets', $metric); $this->count($projectId, 'buckets', $metric);
$this->foreachDocument($project, 'buckets', [], function ($bucket) use (&$projectFilesCount, &$projectFilesTotal, $project, $database) { $this->foreachDocument($projectId, 'buckets', [], function ($bucket) use (&$projectFilesCount, &$projectFilesTotal, $projectId,) {
$metric = "files.{$bucket->getId()}.count.total"; $metric = "files.{$bucket->getId()}.count.total";
$count = $this->count($database, $project->getId(), 'bucket_' . $bucket->getInternalId(), $metric); $count = $this->count($projectId, 'bucket_' . $bucket->getInternalId(), $metric);
$projectFilesCount += $count; $projectFilesCount += $count;
$metric = "files.{$bucket->getId()}.storage.size"; $metric = "files.{$bucket->getId()}.storage.size";
$sum = $this->sum($database, $project->getId(), 'bucket_' . $bucket->getInternalId(), 'sizeOriginal', $metric); $sum = $this->sum($projectId, 'bucket_' . $bucket->getInternalId(), 'sizeOriginal', $metric);
$projectFilesTotal += $sum; $projectFilesTotal += $sum;
}); });
$this->createPerPeriodMetric($database, $project->getId(), 'files.$all.count.total', $projectFilesCount); $this->createPerPeriodMetric($projectId, 'files.$all.count.total', $projectFilesCount);
$this->createPerPeriodMetric($database, $project->getId(), 'files.$all.storage.size', $projectFilesTotal); $this->createPerPeriodMetric($projectId, 'files.$all.storage.size', $projectFilesTotal);
$deploymentsTotal = $this->deploymentsTotal($database, $project->getId()); $deploymentsTotal = $this->deploymentsTotal($projectId);
$this->createPerPeriodMetric($database, $project->getId(), 'project.$all.storage.size', $projectFilesTotal + $deploymentsTotal); $this->createPerPeriodMetric($projectId, 'project.$all.storage.size', $projectFilesTotal + $deploymentsTotal);
} }
/** /**
@ -312,39 +305,38 @@ class Database extends Calculator
* Metrics: databases.$all.count.total, collections.$all.count.total, collections.databaseId.count.total, * Metrics: databases.$all.count.total, collections.$all.count.total, collections.databaseId.count.total,
* documents.$all.count.all, documents.databaseId.count.total, documents.databaseId/collectionId.count.total * documents.$all.count.all, documents.databaseId.count.total, documents.databaseId/collectionId.count.total
* *
* @param UtopiaDatabase $database * @param string $projectId
* @param Document $project
* *
* @return void * @return void
* @throws Authorization * @throws Authorization
* @throws Structure * @throws Structure
*/ */
private function databaseStats(UtopiaDatabase $database, Document $project): void private function databaseStats(string $projectId): void
{ {
$projectDocumentsCount = 0; $projectDocumentsCount = 0;
$projectCollectionsCount = 0; $projectCollectionsCount = 0;
$this->count($database, $project->getId(), 'databases', 'databases.$all.count.total'); $this->count($projectId, 'databases', 'databases.$all.count.total');
$this->foreachDocument($project, 'databases', [], function ($database) use (&$projectDocumentsCount, &$projectCollectionsCount, $project) { $this->foreachDocument($projectId, 'databases', [], function ($database) use (&$projectDocumentsCount, &$projectCollectionsCount, $projectId) {
$metric = "collections.{$database->getId()}.count.total"; $metric = "collections.{$database->getId()}.count.total";
$count = $this->count($database, $project->getId(), 'database_' . $database->getInternalId(), $metric); $count = $this->count($projectId, 'database_' . $database->getInternalId(), $metric);
$projectCollectionsCount += $count; $projectCollectionsCount += $count;
$databaseDocumentsCount = 0; $databaseDocumentsCount = 0;
$this->foreachDocument($project, 'database_' . $database->getInternalId(), [], function ($collection) use (&$projectDocumentsCount, &$databaseDocumentsCount, $project, $database) { $this->foreachDocument($projectId, 'database_' . $database->getInternalId(), [], function ($collection) use (&$projectDocumentsCount, &$databaseDocumentsCount, $projectId, $database) {
$metric = "documents.{$database->getId()}/{$collection->getId()}.count.total"; $metric = "documents.{$database->getId()}/{$collection->getId()}.count.total";
$count = $this->count($database, $project->getId(), 'database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $metric); $count = $this->count($projectId, 'database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $metric);
$projectDocumentsCount += $count; $projectDocumentsCount += $count;
$databaseDocumentsCount += $count; $databaseDocumentsCount += $count;
}); });
$this->createPerPeriodMetric($database, $project->getId(), "documents.{$database->getId()}.count.total", $databaseDocumentsCount); $this->createPerPeriodMetric($projectId, "documents.{$database->getId()}.count.total", $databaseDocumentsCount);
}); });
$this->createPerPeriodMetric($database, $project->getId(), 'collections.$all.count.total', $projectCollectionsCount); $this->createPerPeriodMetric($projectId, 'collections.$all.count.total', $projectCollectionsCount);
$this->createPerPeriodMetric($database, $project->getId(), 'documents.$all.count.total', $projectDocumentsCount); $this->createPerPeriodMetric($projectId, 'documents.$all.count.total', $projectDocumentsCount);
} }
/** /**
@ -357,12 +349,12 @@ class Database extends Calculator
*/ */
public function collect(): void public function collect(): void
{ {
$this->foreachDocument(new Document(['$id' => 'console']), 'projects', [], function (Document $project) { $this->foreachDocument('console', 'projects', [], function (Document $project) {
$database = call_user_func($this->getProjectDB, $project); $projectId = $project->getInternalId();
$this->usersStats($database, $project->getId());
$this->databaseStats($database, $project); $this->usersStats($projectId);
$this->storageStats($database, $project); $this->databaseStats($projectId);
$this->register->get('pools')->reclaim(); $this->storageStats($projectId);
}); });
} }
} }

11
src/Appwrite/Usage/Calculators/TimeSeries.php
View file

@ -14,7 +14,6 @@ class TimeSeries extends Calculator
protected Database $database; protected Database $database;
protected $errorHandler; protected $errorHandler;
private array $latestTime = []; private array $latestTime = [];
private mixed $getProjectDB;
// all the mertics that we are collecting // all the mertics that we are collecting
protected array $metrics = [ protected array $metrics = [
@ -279,11 +278,10 @@ class TimeSeries extends Calculator
'startTime' => '-24 hours', 'startTime' => '-24 hours',
]; ];
public function __construct(Database $database, InfluxDatabase $influxDB, callable $getProjectDB, callable $errorHandler = null) public function __construct(Database $database, InfluxDatabase $influxDB, callable $errorHandler = null)
{ {
$this->database = $database; $this->database = $database;
$this->influxDB = $influxDB; $this->influxDB = $influxDB;
$this->getProjectDB = $getProjectDB;
$this->errorHandler = $errorHandler; $this->errorHandler = $errorHandler;
} }
@ -303,11 +301,12 @@ class TimeSeries extends Calculator
private function createOrUpdateMetric(string $projectId, string $time, string $period, string $metric, int $value, int $type): void private function createOrUpdateMetric(string $projectId, string $time, string $period, string $metric, int $value, int $type): void
{ {
$id = \md5("{$time}_{$period}_{$metric}"); $id = \md5("{$time}_{$period}_{$metric}");
$this->database->setNamespace('_console');
$project = $this->database->getDocument('projects', $projectId); $project = $this->database->getDocument('projects', $projectId);
$database = call_user_func($this->getProjectDB, $project); $this->database->setNamespace('_' . $project->getInternalId());
try { try {
$document = $database->getDocument('stats', $id); $document = $this->database->getDocument('stats', $id);
if ($document->isEmpty()) { if ($document->isEmpty()) {
$this->database->createDocument('stats', new Document([ $this->database->createDocument('stats', new Document([
'$id' => $id, '$id' => $id,
@ -318,7 +317,7 @@ class TimeSeries extends Calculator
'type' => $type, 'type' => $type,
])); ]));
} else { } else {
$database->updateDocument( $this->database->updateDocument(
'stats', 'stats',
$document->getId(), $document->getId(),
$document->setAttribute('value', $value) $document->setAttribute('value', $value)

3
tests/e2e/General/HTTPTest.php
View file

@ -163,8 +163,7 @@ class HTTPTest extends Scope
$response['body'] = json_decode($response['body'], true); $response['body'] = json_decode($response['body'], true);
$this->assertEquals(200, $response['headers']['status-code']); $this->assertEquals(200, $response['headers']['status-code']);
// looks like recent change in the validator $this->assertEmpty($response['body']['schemaValidationMessages']);
$this->assertTrue(empty($response['body']['schemaValidationMessages']));
} }
} }