CodeMirror/appwrite
1
0
Fork 0
mirror of synced 2024-07-04 14:10:33 +12:00
Code Issues Projects Releases Wiki Activity

Merge branch 'feat-db-pools' of github.com:appwrite/appwrite into feat-db-pools

Browse source
This commit is contained in:
Christy Jacob 2022-11-23 03:03:19 +05:30
parent 5730c3deb6 e7a51fac74
commit 2fcaa02226
15 changed files with 173 additions and 132 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
.env
View file

@ -1,6 +1,6 @@
_APP_ENV=development
_APP_LOCALE=en
_APP_WORKER_PER_CORE=6
_APP_WORKER_PER_CORE=2
_APP_CONSOLE_WHITELIST_ROOT=disabled
_APP_CONSOLE_WHITELIST_EMAILS=
_APP_CONSOLE_WHITELIST_IPS=
@ -25,6 +25,7 @@ _APP_DB_USER=user
_APP_DB_PASS=password
_APP_DB_ROOT_PASS=rootsecretpassword
_APP_CONNECTIONS_MAX=251
_APP_POOL_CLIENTS=14
_APP_CONNECTIONS_DB_PROJECT=db_fra1_02=mariadb://user:password@mariadb:3306/appwrite
_APP_CONNECTIONS_DB_CONSOLE=db_fra1_01=mariadb://user:password@mariadb:3306/appwrite
_APP_CONNECTIONS_CACHE=redis_fra1_01=redis://redis:6379
@ -53,6 +54,7 @@ _APP_FUNCTIONS_BUILD_TIMEOUT=900
_APP_FUNCTIONS_CPUS=1
_APP_FUNCTIONS_MEMORY=512
_APP_FUNCTIONS_INACTIVE_THRESHOLD=600
_APP_FUNCTIONS_MAINTENANCE_INTERVAL=600
_APP_FUNCTIONS_RUNTIMES_NETWORK=runtimes
_APP_EXECUTOR_SECRET=your-secret-key
_APP_EXECUTOR_HOST=http://exc1/v1
@ -63,9 +65,9 @@ _APP_MAINTENANCE_RETENTION_EXECUTION=1209600
_APP_MAINTENANCE_RETENTION_ABUSE=86400
_APP_MAINTENANCE_RETENTION_AUDIT=1209600
_APP_MAINTENANCE_RETENTION_SCHEDULES=86400
_APP_USAGE_TIMESERIES_INTERVAL=20
_APP_MAINTENANCE_RETENTION_USAGE_HOURLY=8640000
_APP_USAGE_STATS=enabled
_APP_USAGE_AGGREGATION_INTERVAL=30
_APP_LOGGING_PROVIDER=
_APP_LOGGING_CONFIG=
_APP_REGION=default

1
.github/workflows/tests.yml vendored
View file

@ -33,6 +33,7 @@ jobs:
docker compose build appwrite
docker compose up -d
sleep 30
- name: Doctor
run: docker compose exec -T appwrite doctor

89
app/cli.php
View file

@ -43,17 +43,44 @@ CLI::setResource('pools', function (Registry $register) {
}, ['register']);
CLI::setResource('dbForConsole', function ($pools, $cache) {
$dbAdapter = $pools
->get('console')
->pop()
->getResource()
;
$sleep = 3;
$maxAttempts = 5;
$attempts = 0;
$ready = false;
$database = new Database($dbAdapter, $cache);
do {
$attempts++;
try {
// Prepare database connection
$dbAdapter = $pools
->get('console')
->pop()
->getResource();
$database->setNamespace('console');
$dbForConsole = new Database($dbAdapter, $cache);
$dbForConsole->setNamespace('console');
return $database;
// Ensure tables exist
$collections = Config::getParam('collections', []);
$last = \array_key_last($collections);
if (!($dbForConsole->exists($dbForConsole->getDefaultDatabase(), $last))) { /** TODO cache ready variable using registry */
throw new Exception('Tables not ready yet.');
}
$ready = true;
} catch (\Exception $err) {
Console::warning($err->getMessage());
$pools->get('console')->reclaim();
sleep($sleep);
}
} while ($attempts < $maxAttempts);
if (!$ready) {
throw new Exception("Console is not ready yet. Please try again later.");
}
return $dbForConsole;
}, ['pools', 'cache']);
CLI::setResource('getProjectDB', function (Group $pools, Database $dbForConsole, $cache) {
@ -165,50 +192,4 @@ $cli
Console::error($error->getMessage());
});
$cli
->init()
->inject('pools')
->inject('cache')
->action(function (Group $pools, Cache $cache) {
$maxAttempts = 5;
$sleep = 3;
$attempts = 0;
$ready = false;
do {
$attempts++;
try {
$pools->get('console')->reclaim();
// Prepare database connection
$dbAdapter = $pools
->get('console')
->pop()
->getResource();
$dbForConsole = new Database($dbAdapter, $cache);
$dbForConsole->setNamespace('console');
// Ensure tables exist
$collections = Config::getParam('collections', []);
$last = \array_key_last($collections);
if (!($dbForConsole->exists($dbForConsole->getDefaultDatabase(), $last))) {
throw new Exception('Tables not ready yet.');
}
$ready = true;
} catch (\Exception $err) {
Console::warning($err->getMessage());
sleep($sleep);
}
} while ($attempts < $maxAttempts);
if (!$ready) {
throw new Exception("Console is not ready yet. Please try again later.");
}
});
$cli->run();

20
app/config/variables.php
View file

@ -877,7 +877,16 @@ return [
'required' => false,
'question' => '',
'filter' => ''
]
],
[
'name' => '_APP_FUNCTIONS_MAINTENANCE_INTERVAL',
'description' => 'Interval how often executor checks for inactive runimes. The default value is 60 seconds.',
'introduction' => '1.2.0',
'default' => '60',
'required' => false,
'question' => '',
'filter' => ''
],
],
],
[
@ -929,6 +938,15 @@ return [
'question' => '',
'filter' => ''
],
[
'name' => '_APP_MAINTENANCE_RETENTION_USAGE_HOURLY',
'description' => 'The maximum duration (in seconds) upto which to retain hourly usage metrics. The default value is 8640000 seconds (100 days).',
'introduction' => '',
'default' => '8640000',
'required' => false,
'question' => '',
'filter' => ''
],
[
'name' => '_APP_MAINTENANCE_RETENTION_SCHEDULES',
'description' => 'Schedules deletion interval ( in seconds ) ',

2
app/console

@ -1 +1 @@
Subproject commit 6aa50b8b96b7941a710fe097f3f4d3a74f1eb823
Subproject commit 4e2cecefb571104f0dbbe5a578729f0e17a10242

14
app/init.php
View file

@ -552,10 +552,16 @@ $register->set('pools', function () {
],
];
$instances = 3; // REST, Realtime, CLI
$workerCount = swoole_cpu_num() * intval(App::getEnv('_APP_WORKER_PER_CORE', 6));
$maxConnections = App::getenv('_APP_CONNECTIONS_MAX', 251);
$instanceConnections = $maxConnections / $instances;
$maxConnections = App::getEnv('_APP_CONNECTIONS_MAX', 151);
$instanceConnections = $maxConnections / App::getEnv('_APP_POOL_CLIENTS', 14);
$multiprocessing = App::getEnv('_APP_SERVER_MULTIPROCESS', 'disabled') === 'enabled';
if ($multiprocessing) {
$workerCount = swoole_cpu_num() * intval(App::getEnv('_APP_WORKER_PER_CORE', 6));
} else {
$workerCount = 1;
}
if ($workerCount > $instanceConnections) {
throw new \Exception('Pool size is too small. Increase the number of allowed database connections or decrease the number of workers.', 500);

35
app/views/install/compose.phtml
View file

@ -139,6 +139,7 @@ services:
- _APP_MAINTENANCE_RETENTION_CACHE
- _APP_MAINTENANCE_RETENTION_ABUSE
- _APP_MAINTENANCE_RETENTION_AUDIT
- _APP_MAINTENANCE_RETENTION_USAGE_HOURLY
- _APP_MAINTENANCE_RETENTION_SCHEDULES
- _APP_SMS_PROVIDER
- _APP_SMS_FROM
@ -483,44 +484,13 @@ services:
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_USAGE_AGGREGATION_INTERVAL
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_INFLUXDB_HOST
- _APP_INFLUXDB_PORT
- _APP_USAGE_TIMESERIES_INTERVAL
- _APP_LOGGING_PROVIDER
- _APP_LOGGING_CONFIG
appwrite-usage-database:
image: <?php echo $organization; ?>/<?php echo $image; ?>:<?php echo $version."\n"; ?>
entrypoint:
- usage
- --type=database
container_name: appwrite-usage-database
<<: *x-logging
restart: unless-stopped
networks:
- appwrite
depends_on:
- influxdb
- mariadb
environment:
- _APP_ENV
- _APP_OPENSSL_KEY_V1
- _APP_DB_HOST
- _APP_DB_PORT
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_INFLUXDB_HOST
- _APP_INFLUXDB_PORT
- _APP_USAGE_TIMESERIES_INTERVAL
- _APP_LOGGING_PROVIDER
- _APP_LOGGING_CONFIG
@ -558,6 +528,7 @@ services:
environment:
- OPR_EXECUTOR_CONNECTION_STORAGE=$_APP_CONNECTIONS_STORAGE
- OPR_EXECUTOR_INACTIVE_TRESHOLD=$_APP_FUNCTIONS_INACTIVE_THRESHOLD
- OPR_EXECUTOR_MAINTENANCE_INTERVAL=$_APP_FUNCTIONS_MAINTENANCE_INTERVAL
- OPR_EXECUTOR_NETWORK=$_APP_FUNCTIONS_RUNTIMES_NETWORK
- OPR_EXECUTOR_DOCKER_HUB_USERNAME=$_APP_DOCKER_HUB_USERNAME
- OPR_EXECUTOR_DOCKER_HUB_PASSWORD=$_APP_DOCKER_HUB_PASSWORD

1
app/workers/builds.php
View file

@ -254,6 +254,7 @@ class BuildsV1 extends Worker
$statsd = $register->get('statsd');
$usage = new Stats($statsd);
$usage
->setParam('projectInternalId', $project->getInternalId())
->setParam('projectId', $project->getId())
->setParam('functionId', $function->getId())
->setParam('builds.{scope}.compute', 1)

1
app/workers/deletes.php
View file

@ -260,6 +260,7 @@ class DeletesV1 extends Worker
{
$this->deleteForProjectIds(function (Document $project) use ($hourlyUsageRetentionDatetime) {
$dbForProject = $this->getProjectDB($project);
// Delete Usage stats
$this->deleteByGroup('stats', [
Query::lessThan('time', $hourlyUsageRetentionDatetime),
Query::equal('period', ['1h']),

6
app/workers/functions.php
View file

@ -152,6 +152,10 @@ Server::setResource('execute', function () {
->setAttribute('status', 'failed')
->setAttribute('statusCode', $th->getCode())
->setAttribute('stderr', $th->getMessage());
Console::error($th->getTraceAsString());
Console::error($th->getFile());
Console::error($th->getLine());
Console::error($th->getMessage());
}
@ -205,7 +209,7 @@ Server::setResource('execute', function () {
$usage
->setParam('projectId', $project->getId())
->setParam('projectInternalId', $project->getInternalId())
->setParam('functionId', $function->getId())
->setParam('functionId', $function->getId()) // TODO: We should use functionInternalId in usage stats
->setParam('executions.{scope}.compute', 1)
->setParam('executionStatus', $execution->getAttribute('status', ''))
->setParam('executionTime', $execution->getAttribute('duration'))

16
composer.lock generated
View file

@ -4,7 +4,7 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
"This file is @generated automatically"
],
"content-hash": "4893e1c13630239fe6a20d1c652eb484",
"content-hash": "7a4830071d4d0c427adc32da23ed1856",
"packages": [
{
"name": "adhocore/jwt",
@ -2356,16 +2356,16 @@
},
{
"name": "utopia-php/pools",
"version": "0.4.1",
"version": "0.4.2",
"source": {
"type": "git",
"url": "https://github.com/utopia-php/pools.git",
"reference": "c8f96a33e7fbf58c1145eb6cf0f2c00cbe319979"
"reference": "d2870ab74b31b7f4027799f082e85122154f8bed"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/utopia-php/pools/zipball/c8f96a33e7fbf58c1145eb6cf0f2c00cbe319979",
"reference": "c8f96a33e7fbf58c1145eb6cf0f2c00cbe319979",
"url": "https://api.github.com/repos/utopia-php/pools/zipball/d2870ab74b31b7f4027799f082e85122154f8bed",
"reference": "d2870ab74b31b7f4027799f082e85122154f8bed",
"shasum": ""
},
"require": {
@ -2401,9 +2401,9 @@
],
"support": {
"issues": "https://github.com/utopia-php/pools/issues",
"source": "https://github.com/utopia-php/pools/tree/0.4.1"
"source": "https://github.com/utopia-php/pools/tree/0.4.2"
},
"time": "2022-11-15T08:55:16+00:00"
"time": "2022-11-22T07:55:45+00:00"
},
{
"name": "utopia-php/preloader",
@ -5403,5 +5403,5 @@
"platform-overrides": {
"php": "8.0"
},
"plugin-api-version": "2.1.0"
"plugin-api-version": "2.3.0"
}

46
docker-compose.yml
View file

@ -93,6 +93,7 @@ services:
- ./public:/usr/src/code/public
- ./src:/usr/src/code/src
- ./dev:/usr/local/dev
depends_on:
- mariadb
- redis
@ -105,6 +106,7 @@ services:
environment:
- _APP_ENV
- _APP_WORKER_PER_CORE
- _APP_SERVER_MULTIPROCESS=enabled
- _APP_LOCALE
- _APP_CONSOLE_WHITELIST_ROOT
- _APP_CONSOLE_WHITELIST_EMAILS
@ -125,6 +127,7 @@ services:
- _APP_DB_USER
- _APP_DB_PASS
- _APP_CONNECTIONS_MAX
- _APP_POOL_CLIENTS
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
@ -204,6 +207,7 @@ services:
environment:
- _APP_ENV
- _APP_WORKER_PER_CORE
- _APP_SERVER_MULTIPROCESS=enabled
- _APP_OPTIONS_ABUSE
- _APP_OPENSSL_KEY_V1
- _APP_DB_HOST
@ -212,6 +216,7 @@ services:
- _APP_DB_USER
- _APP_DB_PASS
- _APP_CONNECTIONS_MAX
- _APP_POOL_CLIENTS
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
@ -275,6 +280,8 @@ services:
environment:
- _APP_ENV
- _APP_WORKER_PER_CORE
- _APP_CONNECTIONS_MAX
- _APP_POOL_CLIENTS
- _APP_OPENSSL_KEY_V1
- _APP_SYSTEM_SECURITY_EMAIL_ADDRESS
- _APP_REDIS_HOST
@ -306,6 +313,8 @@ services:
environment:
- _APP_ENV
- _APP_WORKER_PER_CORE
- _APP_CONNECTIONS_MAX
- _APP_POOL_CLIENTS
- _APP_OPENSSL_KEY_V1
- _APP_DB_HOST
- _APP_DB_PORT
@ -342,6 +351,8 @@ services:
environment:
- _APP_ENV
- _APP_WORKER_PER_CORE
- _APP_CONNECTIONS_MAX
- _APP_POOL_CLIENTS
- _APP_OPENSSL_KEY_V1
- _APP_DB_HOST
- _APP_DB_PORT
@ -375,6 +386,8 @@ services:
environment:
- _APP_ENV
- _APP_WORKER_PER_CORE
- _APP_CONNECTIONS_MAX
- _APP_POOL_CLIENTS
- _APP_OPENSSL_KEY_V1
- _APP_EXECUTOR_SECRET
- _APP_EXECUTOR_HOST
@ -413,6 +426,8 @@ services:
environment:
- _APP_ENV
- _APP_WORKER_PER_CORE
- _APP_CONNECTIONS_MAX
- _APP_POOL_CLIENTS
- _APP_OPENSSL_KEY_V1
- _APP_DOMAIN
- _APP_DOMAIN_TARGET
@ -450,6 +465,8 @@ services:
environment:
- _APP_ENV
- _APP_WORKER_PER_CORE
- _APP_CONNECTIONS_MAX
- _APP_POOL_CLIENTS
- _APP_OPENSSL_KEY_V1
- _APP_DB_HOST
- _APP_DB_PORT
@ -488,6 +505,8 @@ services:
environment:
- _APP_ENV
- _APP_WORKER_PER_CORE
- _APP_CONNECTIONS_MAX
- _APP_POOL_CLIENTS
- _APP_OPENSSL_KEY_V1
- _APP_SYSTEM_EMAIL_NAME
- _APP_SYSTEM_EMAIL_ADDRESS
@ -519,6 +538,8 @@ services:
environment:
- _APP_ENV
- _APP_WORKER_PER_CORE
- _APP_CONNECTIONS_MAX
- _APP_POOL_CLIENTS
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
@ -544,6 +565,8 @@ services:
environment:
- _APP_ENV
- _APP_WORKER_PER_CORE
- _APP_CONNECTIONS_MAX
- _APP_POOL_CLIENTS
- _APP_DOMAIN
- _APP_DOMAIN_TARGET
- _APP_OPENSSL_KEY_V1
@ -564,6 +587,7 @@ services:
- _APP_MAINTENANCE_RETENTION_CACHE
- _APP_MAINTENANCE_RETENTION_ABUSE
- _APP_MAINTENANCE_RETENTION_AUDIT
- _APP_MAINTENANCE_RETENTION_USAGE_HOURLY
- _APP_MAINTENANCE_RETENTION_SCHEDULES
appwrite-usage:
@ -583,6 +607,8 @@ services:
environment:
- _APP_ENV
- _APP_WORKER_PER_CORE
- _APP_CONNECTIONS_MAX
- _APP_POOL_CLIENTS
- _APP_OPENSSL_KEY_V1
- _APP_DB_HOST
- _APP_DB_PORT
@ -591,6 +617,7 @@ services:
- _APP_DB_PASS
- _APP_INFLUXDB_HOST
- _APP_INFLUXDB_PORT
- _APP_USAGE_AGGREGATION_INTERVAL
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
@ -598,7 +625,6 @@ services:
- _APP_CONNECTIONS_DB_CONSOLE
- _APP_CONNECTIONS_DB_PROJECT
- _APP_CONNECTIONS_CACHE
- _APP_USAGE_TIMESERIES_INTERVAL
- _APP_LOGGING_PROVIDER
- _APP_LOGGING_CONFIG
@ -618,6 +644,8 @@ services:
environment:
- _APP_ENV
- _APP_WORKER_PER_CORE
- _APP_CONNECTIONS_MAX
- _APP_POOL_CLIENTS
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
@ -638,7 +666,7 @@ services:
hostname: exc1
<<: *x-logging
stop_signal: SIGINT
image: openruntimes/executor:0.1.4
image: openruntimes/executor:0.1.6
networks:
- appwrite
- runtimes
@ -650,6 +678,7 @@ services:
environment:
- OPR_EXECUTOR_CONNECTION_STORAGE=$_APP_CONNECTIONS_STORAGE
- OPR_EXECUTOR_INACTIVE_TRESHOLD=$_APP_FUNCTIONS_INACTIVE_THRESHOLD
- OPR_EXECUTOR_MAINTENANCE_INTERVAL=$_APP_FUNCTIONS_MAINTENANCE_INTERVAL
- OPR_EXECUTOR_NETWORK=$_APP_FUNCTIONS_RUNTIMES_NETWORK
- OPR_EXECUTOR_DOCKER_HUB_USERNAME=$_APP_DOCKER_HUB_USERNAME
- OPR_EXECUTOR_DOCKER_HUB_PASSWORD=$_APP_DOCKER_HUB_PASSWORD
@ -769,6 +798,19 @@ services:
- 9506:8080
networks:
- appwrite
# appwrite-volume-sync:
# entrypoint: volume-sync
# <<: *x-logging
# container_name: appwrite-volume-sync
# image: appwrite-dev
# command:
# - --source=/data/src/ --destination=/data/dest/ --interval=10
# networks:
# - appwrite
# # volumes: # Mount the rsync source and destination directories
# # - /nfs/config:/data/src
# # - /storage/config:/data/dest
# redis-commander:
# image: rediscommander/redis-commander:latest

2
src/Appwrite/Platform/Tasks/Maintenance.php
View file

@ -136,7 +136,7 @@ class Maintenance extends Action
$cacheRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_CACHE', '2592000'); // 30 days
$schedulesDeletionRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_SCHEDULES', '86400'); // 1 Day
Console::loop(function () use ($interval, $executionLogsRetention, $abuseLogsRetention, $auditLogRetention, $usageStatsRetentionHourly, $cacheRetention, $schedulesDeletionRetention, $dbForConsole) {
Console::loop(function () use ($interval, $executionLogsRetention, $abuseLogsRetention, $auditLogRetention, $cacheRetention, $schedulesDeletionRetention, $usageStatsRetentionHourly, $dbForConsole) {
$time = DateTime::now();
Console::info("[{$time}] Notifying workers with maintenance tasks every {$interval} seconds");

36
src/Appwrite/Platform/Tasks/Usage.php
View file

@ -7,9 +7,9 @@ use InfluxDB\Database as InfluxDatabase;
use Utopia\App;
use Utopia\CLI\Console;
use Utopia\Database\Database as UtopiaDatabase;
use Utopia\Validator\WhiteList;
use Throwable;
use Utopia\Platform\Action;
use Utopia\Registry\Registry;
class Usage extends Action
{
@ -22,18 +22,28 @@ class Usage extends Action
{
$this
->desc('Schedules syncing data from influxdb to Appwrite console db')
->param('type', 'timeseries', new WhiteList(['timeseries', 'database']))
->inject('dbForConsole')
->inject('influxdb')
->inject('register')
->inject('getProjectDB')
->inject('logError')
->callback(fn ($type, $dbForConsole, $influxDB, $logError) => $this->action($type, $dbForConsole, $influxDB, $logError));
->callback(fn ($dbForConsole, $influxDB, $register, $getProjectDB, $logError) => $this->action($dbForConsole, $influxDB, $register, $getProjectDB, $logError));
}
protected function aggregateTimeseries(UtopiaDatabase $database, InfluxDatabase $influxDB, callable $logError): void
{
$interval = (int) App::getEnv('_APP_USAGE_TIMESERIES_INTERVAL', '30'); // 30 seconds (by default)
}
public function action(UtopiaDatabase $dbForConsole, InfluxDatabase $influxDB, Registry $register, callable $getProjectDB, callable $logError)
{
Console::title('Usage Aggregation V1');
Console::success(APP_NAME . ' usage aggregation process v1 has started');
$errorLogger = fn(Throwable $error, string $action = 'syncUsageStats') => $logError($error, "usage", $action);
$interval = (int) App::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '30'); // 30 seconds (by default)
$region = App::getEnv('region', 'default');
$usage = new TimeSeries($region, $database, $influxDB, $logError);
$usage = new TimeSeries($region, $dbForConsole, $influxDB, $getProjectDB, $register, $errorLogger);
Console::loop(function () use ($interval, $usage) {
$now = date('d-m-Y H:i:s', time());
@ -47,20 +57,4 @@ class Usage extends Action
Console::info("[{$now}] Aggregation took {$loopTook} seconds");
}, $interval);
}
public function action(string $type, UtopiaDatabase $dbForConsole, InfluxDatabase $influxDB, callable $logError)
{
Console::title('Usage Aggregation V1');
Console::success(APP_NAME . ' usage aggregation process v1 has started');
$errorLogger = fn(Throwable $error, string $action = 'syncUsageStats') => $logError($error, "usage", $action);
switch ($type) {
case 'timeseries':
$this->aggregateTimeseries($dbForConsole, $influxDB, $errorLogger);
break;
default:
Console::error("Unsupported usage aggregation type");
}
}
}

30
src/Appwrite/Usage/Calculators/TimeSeries.php
View file

@ -8,6 +8,7 @@ use Utopia\Database\Database;
use Utopia\Database\Document;
use InfluxDB\Database as InfluxDatabase;
use DateTime;
use Utopia\Registry\Registry;
class TimeSeries extends Calculator
{
@ -32,6 +33,20 @@ class TimeSeries extends Calculator
*/
protected $errorHandler;
/**
* Callback to get project DB
*
* @var callable
*/
protected mixed $getProjectDB;
/**
* Registry
*
* @var Registry
*/
protected Registry $register;
/**
* Latest times for metric that was synced to the database
*
@ -382,11 +397,13 @@ class TimeSeries extends Calculator
]
];
public function __construct(string $region, Database $database, InfluxDatabase $influxDB, callable $errorHandler = null)
public function __construct(string $region, Database $database, InfluxDatabase $influxDB, callable $getProjectDB, Registry $register, callable $errorHandler = null)
{
parent::__construct($region);
$this->database = $database;
$this->influxDB = $influxDB;
$this->getProjectDB = $getProjectDB;
$this->register = $register;
$this->errorHandler = $errorHandler;
}
@ -406,12 +423,13 @@ class TimeSeries extends Calculator
private function createOrUpdateMetric(string $projectId, string $time, string $period, string $metric, int $value, int $type): void
{
$id = \md5("{$time}_{$period}_{$metric}");
$this->database->setNamespace('_' . $projectId);
$project = $this->database->getDocument('projects', $projectId);
$database = call_user_func($this->getProjectDB, $project);
try {
$document = $this->database->getDocument('stats', $id);
$document = $database->getDocument('stats', $id);
if ($document->isEmpty()) {
$this->database->createDocument('stats', new Document([
$database->createDocument('stats', new Document([
'$id' => $id,
'period' => $period,
'time' => $time,
@ -421,7 +439,7 @@ class TimeSeries extends Calculator
'region' => $this->region,
]));
} else {
$this->database->updateDocument(
$database->updateDocument(
'stats',
$document->getId(),
$document->setAttribute('value', $value)
@ -434,6 +452,8 @@ class TimeSeries extends Calculator
throw $e;
}
}
$this->register->get('pools')->reclaim();
}
/**