1
0
Fork 0
mirror of synced 2024-09-29 17:01:37 +13:00

addressing comments

This commit is contained in:
shimon 2023-02-02 13:55:23 +02:00
parent 82f8fbdff8
commit abdd5f016a
9 changed files with 295 additions and 5367 deletions

@ -1 +1 @@
Subproject commit d64fc2c3f981a6213cf28812364fcf92ac3f61af
Subproject commit fae048b91787d0b372c13caf27f14fc8b780ef60

View file

@ -2385,6 +2385,74 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu
$response->noContent();
});
App::get('/v1/databases/usage')
->desc('Get usage stats for the database')
->groups(['api', 'database'])
->label('scope', 'collections.read')
->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'getUsage')
->label('sdk.response.code', Response::STATUS_CODE_OK)
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
->label('sdk.response.model', Response::MODEL_USAGE_DATABASES)
->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), '`Date range.', true)
->inject('response')
->inject('dbForProject')
->action(function (string $range, Response $response, Database $dbForProject) {
$periods = Config::getParam('usage', []);
$stats = $usage = [];
$days = $periods[$range];
$metrics = [
'databases',
'collections',
'documents',
];
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
foreach ($metrics as $metric) {
$limit = $days['limit'];
$period = $days['period'];
$results = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric] = [];
foreach ($results as $result) {
$stats[$metric][$result->getAttribute('time')] = [
'value' => $result->getAttribute('value'),
];
}
}
});
$format = match ($days['period']) {
'1h' => 'Y-m-d\TH:00:00.000P',
'1d' => 'Y-m-d\T00:00:00.000P',
};
foreach ($metrics as $metric) {
$usage[$metric] = [];
$leap = time() - ($days['limit'] * $days['factor']);
while ($leap < time()) {
$leap += $days['factor'];
$formatDate = date($format, $leap);
$usage[$metric][] = [
'value' => $stats[$metric][$formatDate]['value'] ?? 0,
'date' => $formatDate,
];
}
}
$response->dynamic(new Document([
'range' => $range,
'databasesCount' => $usage[$metrics[0]],
'collectionsCount' => $usage[$metrics[1]],
'documentsCount' => $usage[$metrics[2]],
]), Response::MODEL_USAGE_DATABASES);
});
App::get('/v1/databases/:databaseId/usage')
->desc('Get usage stats for the database')
->groups(['api', 'database'])
@ -2535,71 +2603,3 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/usage')
'documentsCount' => $usage[$metrics[0]],
]), Response::MODEL_USAGE_COLLECTION);
});
App::get('/v1/databases/usage')
->desc('Get usage stats for the database')
->groups(['api', 'database'])
->label('scope', 'collections.read')
->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'getUsage')
->label('sdk.response.code', Response::STATUS_CODE_OK)
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
->label('sdk.response.model', Response::MODEL_USAGE_DATABASES)
->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), '`Date range.', true)
->inject('response')
->inject('dbForProject')
->action(function (string $range, Response $response, Database $dbForProject) {
$periods = Config::getParam('usage', []);
$stats = $usage = [];
$days = $periods[$range];
$metrics = [
'databases',
'collections',
'documents',
];
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
foreach ($metrics as $metric) {
$limit = $days['limit'];
$period = $days['period'];
$results = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric] = [];
foreach ($results as $result) {
$stats[$metric][$result->getAttribute('time')] = [
'value' => $result->getAttribute('value'),
];
}
}
});
$format = match ($days['period']) {
'1h' => 'Y-m-d\TH:00:00.000P',
'1d' => 'Y-m-d\T00:00:00.000P',
};
foreach ($metrics as $metric) {
$usage[$metric] = [];
$leap = time() - ($days['limit'] * $days['factor']);
while ($leap < time()) {
$leap += $days['factor'];
$formatDate = date($format, $leap);
$usage[$metric][] = [
'value' => $stats[$metric][$formatDate]['value'] ?? 0,
'date' => $formatDate,
];
}
}
$response->dynamic(new Document([
'range' => $range,
'databasesCount' => $usage[$metrics[0]],
'collectionsCount' => $usage[$metrics[1]],
'documentsCount' => $usage[$metrics[2]],
]), Response::MODEL_USAGE_DATABASES);
});

View file

@ -210,6 +210,168 @@ App::get('/v1/functions/:functionId')
$response->dynamic($function, Response::MODEL_FUNCTION);
});
App::get('/v1/functions/:functionId/usage')
->desc('Get Function Usage')
->groups(['api', 'functions'])
->label('scope', 'functions.read')
->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
->label('sdk.namespace', 'functions')
->label('sdk.method', 'getFunctionUsage')
->label('sdk.response.code', Response::STATUS_CODE_OK)
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
->label('sdk.response.model', Response::MODEL_USAGE_FUNCTIONS)
->param('functionId', '', new UID(), 'Function ID.')
->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d']), 'Date range.', true)
->inject('response')
->inject('dbForProject')
->action(function (string $functionId, string $range, Response $response, Database $dbForProject) {
$function = $dbForProject->getDocument('functions', $functionId);
if ($function->isEmpty()) {
throw new Exception(Exception::FUNCTION_NOT_FOUND);
}
$periods = Config::getParam('usage', []);
$stats = $usage = [];
$days = $periods[$range];
$metrics = [
'functions.' . $function->getInternalId() . '.deployments',
'functions.' . $function->getInternalId() . '.deployments.storage',
$function->getInternalId() . '.builds',
$function->getInternalId() . '.builds.storage',
$function->getInternalId() . '.builds.compute',
$function->getInternalId() . '.executions',
$function->getInternalId() . '.executions.compute',
];
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
foreach ($metrics as $metric) {
$limit = $days['limit'];
$period = $days['period'];
$results = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric] = [];
foreach ($results as $result) {
$stats[$metric][$result->getAttribute('time')] = [
'value' => $result->getAttribute('value'),
];
}
}
});
$format = match ($days['period']) {
'1h' => 'Y-m-d\TH:00:00.000P',
'1d' => 'Y-m-d\T00:00:00.000P',
};
foreach ($metrics as $metric) {
$usage[$metric] = [];
$leap = time() - ($days['limit'] * $days['factor']);
while ($leap < time()) {
$leap += $days['factor'];
$formatDate = date($format, $leap);
$usage[$metric][] = [
'value' => $stats[$metric][$formatDate]['value'] ?? 0,
'date' => $formatDate,
];
}
}
$response->dynamic(new Document([
'range' => $range,
'deployments' => $usage[$metrics[0]],
'deploymentsStorage' => $usage[$metrics[1]],
'builds' => $usage[$metrics[2]],
'buildsStorage' => $usage[$metrics[3]],
'buildsCompute' => $usage[$metrics[4]],
'executions' => $usage[$metrics[5]],
'executionsCompute' => $usage[$metrics[6]],
]), Response::MODEL_USAGE_FUNCTION);
});
App::get('/v1/functions/usage')
->desc('Get Functions Usage')
->groups(['api', 'functions'])
->label('scope', 'functions.read')
->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
->label('sdk.namespace', 'functions')
->label('sdk.method', 'getUsage')
->label('sdk.response.code', Response::STATUS_CODE_OK)
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
->label('sdk.response.model', Response::MODEL_USAGE_FUNCTIONS)
->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d']), 'Date range.', true)
->inject('response')
->inject('dbForProject')
->action(function (string $range, Response $response, Database $dbForProject) {
$periods = Config::getParam('usage', []);
$stats = $usage = [];
$days = $periods[$range];
$metrics = [
'functions',
'deployments',
'deployments.storage',
'builds',
'builds.storage',
'builds.compute',
'executions',
'executions.compute',
];
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
foreach ($metrics as $metric) {
$limit = $days['limit'];
$period = $days['period'];
$results = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric] = [];
foreach ($results as $result) {
$stats[$metric][$result->getAttribute('time')] = [
'value' => $result->getAttribute('value'),
];
}
}
});
$format = match ($days['period']) {
'1h' => 'Y-m-d\TH:00:00.000P',
'1d' => 'Y-m-d\T00:00:00.000P',
};
foreach ($metrics as $metric) {
$usage[$metric] = [];
$leap = time() - ($days['limit'] * $days['factor']);
while ($leap < time()) {
$leap += $days['factor'];
$formatDate = date($format, $leap);
$usage[$metric][] = [
'value' => $stats[$metric][$formatDate]['value'] ?? 0,
'date' => $formatDate,
];
}
}
$response->dynamic(new Document([
'range' => $range,
'functions' => $usage[$metrics[0]],
'deployments' => $usage[$metrics[1]],
'deploymentsStorage' => $usage[$metrics[2]],
'builds' => $usage[$metrics[3]],
'buildsStorage' => $usage[$metrics[4]],
'buildsCompute' => $usage[$metrics[5]],
'executions' => $usage[$metrics[6]],
'executionsCompute' => $usage[$metrics[7]],
]), Response::MODEL_USAGE_FUNCTIONS);
});
App::put('/v1/functions/:functionId')
->groups(['api', 'functions'])
->desc('Update Function')
@ -1362,165 +1524,3 @@ App::delete('/v1/functions/:functionId/variables/:variableId')
$response->noContent();
});
App::get('/v1/functions/:functionId/usage')
->desc('Get Function Usage')
->groups(['api', 'functions'])
->label('scope', 'functions.read')
->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
->label('sdk.namespace', 'functions')
->label('sdk.method', 'getFunctionUsage')
->label('sdk.response.code', Response::STATUS_CODE_OK)
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
->label('sdk.response.model', Response::MODEL_USAGE_FUNCTIONS)
->param('functionId', '', new UID(), 'Function ID.')
->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d']), 'Date range.', true)
->inject('response')
->inject('dbForProject')
->action(function (string $functionId, string $range, Response $response, Database $dbForProject) {
$function = $dbForProject->getDocument('functions', $functionId);
if ($function->isEmpty()) {
throw new Exception(Exception::FUNCTION_NOT_FOUND);
}
$periods = Config::getParam('usage', []);
$stats = $usage = [];
$days = $periods[$range];
$metrics = [
'functions.' . $function->getInternalId() . '.deployments',
'functions.' . $function->getInternalId() . '.deployments.storage',
$function->getInternalId() . '.builds',
$function->getInternalId() . '.builds.storage',
$function->getInternalId() . '.builds.compute',
$function->getInternalId() . '.executions',
$function->getInternalId() . '.executions.compute',
];
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
foreach ($metrics as $metric) {
$limit = $days['limit'];
$period = $days['period'];
$results = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric] = [];
foreach ($results as $result) {
$stats[$metric][$result->getAttribute('time')] = [
'value' => $result->getAttribute('value'),
];
}
}
});
$format = match ($days['period']) {
'1h' => 'Y-m-d\TH:00:00.000P',
'1d' => 'Y-m-d\T00:00:00.000P',
};
foreach ($metrics as $metric) {
$usage[$metric] = [];
$leap = time() - ($days['limit'] * $days['factor']);
while ($leap < time()) {
$leap += $days['factor'];
$formatDate = date($format, $leap);
$usage[$metric][] = [
'value' => $stats[$metric][$formatDate]['value'] ?? 0,
'date' => $formatDate,
];
}
}
$response->dynamic(new Document([
'range' => $range,
'deployments' => $usage[$metrics[0]],
'deploymentsStorage' => $usage[$metrics[1]],
'builds' => $usage[$metrics[2]],
'buildsStorage' => $usage[$metrics[3]],
'buildsCompute' => $usage[$metrics[4]],
'executions' => $usage[$metrics[5]],
'executionsCompute' => $usage[$metrics[6]],
]), Response::MODEL_USAGE_FUNCTION);
});
App::get('/v1/functions/usage')
->desc('Get Functions Usage')
->groups(['api', 'functions'])
->label('scope', 'functions.read')
->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
->label('sdk.namespace', 'functions')
->label('sdk.method', 'getUsage')
->label('sdk.response.code', Response::STATUS_CODE_OK)
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
->label('sdk.response.model', Response::MODEL_USAGE_FUNCTIONS)
->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d']), 'Date range.', true)
->inject('response')
->inject('dbForProject')
->action(function (string $range, Response $response, Database $dbForProject) {
$periods = Config::getParam('usage', []);
$stats = $usage = [];
$days = $periods[$range];
$metrics = [
'functions',
'deployments',
'deployments.storage',
'builds',
'builds.storage',
'builds.compute',
'executions',
'executions.compute',
];
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
foreach ($metrics as $metric) {
$limit = $days['limit'];
$period = $days['period'];
$results = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric] = [];
foreach ($results as $result) {
$stats[$metric][$result->getAttribute('time')] = [
'value' => $result->getAttribute('value'),
];
}
}
});
$format = match ($days['period']) {
'1h' => 'Y-m-d\TH:00:00.000P',
'1d' => 'Y-m-d\T00:00:00.000P',
};
foreach ($metrics as $metric) {
$usage[$metric] = [];
$leap = time() - ($days['limit'] * $days['factor']);
while ($leap < time()) {
$leap += $days['factor'];
$formatDate = date($format, $leap);
$usage[$metric][] = [
'value' => $stats[$metric][$formatDate]['value'] ?? 0,
'date' => $formatDate,
];
}
}
$response->dynamic(new Document([
'range' => $range,
'functions' => $usage[$metrics[0]],
'deployments' => $usage[$metrics[1]],
'deploymentsStorage' => $usage[$metrics[2]],
'builds' => $usage[$metrics[3]],
'buildsStorage' => $usage[$metrics[4]],
'buildsCompute' => $usage[$metrics[5]],
'executions' => $usage[$metrics[6]],
'executionsCompute' => $usage[$metrics[7]],
]), Response::MODEL_USAGE_FUNCTIONS);
});

View file

@ -175,6 +175,8 @@ const APP_AUTH_TYPE_ADMIN = 'Admin';
// Response related
const MAX_OUTPUT_CHUNK_SIZE = 2 * 1024 * 1024; // 2MB
$register = new Registry();
App::setMode(App::getEnv('_APP_ENV', App::MODE_TYPE_PRODUCTION));

View file

@ -430,42 +430,6 @@ services:
- _APP_LOGGING_PROVIDER
- _APP_LOGGING_CONFIG
appwrite-worker-usage:
image: <?php echo $organization; ?>/<?php echo $image; ?>:<?php echo $version."\n"; ?>
entrypoint: worker-usage
<<: *x-logging
container_name: appwrite-worker-usage
networks:
- appwrite
volumes:
- ./app:/usr/src/code/app
- ./src:/usr/src/code/src
depends_on:
- redis
- mariadb
environment:
- _APP_ENV
- _APP_WORKER_PER_CORE
- _APP_CONNECTIONS_MAX
- _APP_POOL_CLIENTS
- _APP_OPENSSL_KEY_V1
- _APP_DB_HOST
- _APP_DB_PORT
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_CONNECTIONS_DB_CONSOLE
- _APP_CONNECTIONS_DB_PROJECT
- _APP_CONNECTIONS_CACHE
- _APP_CONNECTIONS_QUEUE
- _APP_USAGE_STATS
- DOCKERHUB_PULL_USERNAME
- DOCKERHUB_PULL_PASSWORD
appwrite-maintenance:
image: <?php echo $organization; ?>/<?php echo $image; ?>:<?php echo $version."\n"; ?>
entrypoint: maintenance
@ -581,6 +545,42 @@ services:
# volumes:
# - appwrite-uploads:/storage/uploads
appwrite-worker-usage:
image: <?php echo $organization; ?>/<?php echo $image; ?>:<?php echo $version."\n"; ?>
entrypoint: worker-usage
<<: *x-logging
container_name: appwrite-worker-usage
networks:
- appwrite
volumes:
- ./app:/usr/src/code/app
- ./src:/usr/src/code/src
depends_on:
- redis
- mariadb
environment:
- _APP_ENV
- _APP_WORKER_PER_CORE
- _APP_CONNECTIONS_MAX
- _APP_POOL_CLIENTS
- _APP_OPENSSL_KEY_V1
- _APP_DB_HOST
- _APP_DB_PORT
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_CONNECTIONS_DB_CONSOLE
- _APP_CONNECTIONS_DB_PROJECT
- _APP_CONNECTIONS_CACHE
- _APP_CONNECTIONS_QUEUE
- _APP_USAGE_STATS
- DOCKERHUB_PULL_USERNAME
- DOCKERHUB_PULL_PASSWORD
networks:
gateway:
name: gateway

5090
composer.lock generated

File diff suppressed because it is too large Load diff

View file

@ -33,15 +33,6 @@ services:
- 8080:80
- 443:443
- 9500:8080
ulimits:
nofile:
soft: 655350
hard: 655350
sysctls:
- net.core.somaxconn=1024
- net.ipv4.tcp_rmem=1024 4096 16384
- net.ipv4.tcp_wmem=1024 4096 16384
- net.ipv4.ip_local_port_range=1025 65535
volumes:
- /var/run/docker.sock:/var/run/docker.sock
- appwrite-config:/storage/config:ro
@ -62,6 +53,7 @@ services:
DEBUG: false
TESTING: true
VERSION: dev
VITE_CONSOLE_MODE: self-hosted
ports:
- 9501:80
networks:

View file

@ -12,7 +12,6 @@ use Appwrite\Platform\Tasks\PatchCreateMissingSchedules;
use Appwrite\Platform\Tasks\SDKs;
use Appwrite\Platform\Tasks\Specs;
use Appwrite\Platform\Tasks\SSL;
use Appwrite\Platform\Tasks\Usage;
use Appwrite\Platform\Tasks\Vars;
use Appwrite\Platform\Tasks\Version;
use Appwrite\Platform\Tasks\VolumeSync;

View file

@ -2,16 +2,20 @@
namespace Appwrite\Platform\Tasks;
use Appwrite\URL\URL;
use Utopia\App;
use Utopia\CLI\Console;
use Appwrite\ClamAV\Network;
use Utopia\Logger\Logger;
use Utopia\Queue\Client;
use Utopia\Storage\Device\Local;
use Utopia\Storage\Storage;
use Utopia\Config\Config;
use Utopia\Domains\Domain;
use Utopia\Platform\Action;
use Utopia\Registry\Registry;
use Utopia\DSN\DSN;
use Utopia\Queue;
class Doctor extends Action
{
@ -187,6 +191,27 @@ class Doctor extends Action
Console::error('🔴 ' . str_pad("SMTP", 47, '.') . 'disconnected');
}
try {
$fallbackForRedis = URL::unparse([
'scheme' => 'redis',
'host' => App::getEnv('_APP_REDIS_HOST', 'redis'),
'port' => App::getEnv('_APP_REDIS_PORT', '6379'),
'user' => App::getEnv('_APP_REDIS_USER', ''),
'pass' => App::getEnv('_APP_REDIS_PASS', ''),
]);
$dsn = App::getEnv('_APP_CONNECTIONS_QUEUE', $fallbackForRedis);
$dsn = explode('=', $dsn);
$dsn = $dsn[1] ?? '';
$dsn = new DSN($dsn);
$connection = new Queue\Connection\Redis($dsn->getHost(), $dsn->getPort());
$client = new Client('v1-usage', $connection);
$client->getQueueSize();
Console::success('🟢 ' . str_pad("Usage queue", 50, '.') . 'connected');
} catch (\Throwable $th) {
Console::error('🔴 ' . str_pad("Usage queue", 47, '.') . 'disconnected');
}
\sleep(0.2);
Console::log('');