CodeMirror/appwrite
1
0
Fork 0
mirror of synced 2024-06-27 18:50:47 +12:00
Code Issues Projects Releases Wiki Activity

addressing comments

Browse source
This commit is contained in:
shimon 2023-02-02 20:16:01 +02:00
parent a30dfd284c
commit 1ac935077b
10 changed files with 108 additions and 128 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
app/controllers/api/databases.php
View file

@ -2404,9 +2404,9 @@ App::get('/v1/databases/usage')
$stats = $usage = [];
$days = $periods[$range];
$metrics = [
'databases',
'collections',
'documents',
METRIC_DATABASES,
METRIC_COLLECTIONS,
METRIC_DOCUMENTS,
];
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
@ -2479,8 +2479,8 @@ App::get('/v1/databases/:databaseId/usage')
$stats = $usage = [];
$days = $periods[$range];
$metrics = [
$database->getInternalId() . '.collections',
$database->getInternalId() . '.documents',
str_replace('{databaseId}', $database->getInternalId(), METRIC_DATABASE_ID_COLLECTIONS),
str_replace('{databaseId}', $database->getInternalId(), METRIC_DATABASE_ID_DOCUMENTS),
];
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
@ -2546,7 +2546,6 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/usage')
->action(function (string $databaseId, string $range, string $collectionId, Response $response, Database $dbForProject) {
$database = $dbForProject->getDocument('databases', $databaseId);
$collectionDocument = $dbForProject->getDocument('database_' . $database->getInternalId(), $collectionId);
$collection = $dbForProject->getCollection('database_' . $database->getInternalId() . '_collection_' . $collectionDocument->getInternalId());
@ -2558,7 +2557,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/usage')
$stats = $usage = [];
$days = $periods[$range];
$metrics = [
$collectionDocument->getInternalId() . '.documents',
str_replace(['{databaseId}', '{collectionId}'], [$database->getInternalId(), $collectionDocument->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS),
];
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {

52
app/controllers/api/functions.php
View file

@ -236,13 +236,13 @@ App::get('/v1/functions/:functionId/usage')
$stats = $usage = [];
$days = $periods[$range];
$metrics = [
'functions.' . $function->getInternalId() . '.deployments',
'functions.' . $function->getInternalId() . '.deployments.storage',
$function->getInternalId() . '.builds',
$function->getInternalId() . '.builds.storage',
$function->getInternalId() . '.builds.compute',
$function->getInternalId() . '.executions',
$function->getInternalId() . '.executions.compute',
str_replace(['{resourceType}', '{resourceInternalId}'], ['functions', $function->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS),
str_replace(['{resourceType}', '{resourceInternalId}'], $function->getInternalId(), METRIC_FUNCTION_ID_STORAGE),
str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS),
str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_STORAGE),
str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE),
str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS),
str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE),
];
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
@ -284,13 +284,13 @@ App::get('/v1/functions/:functionId/usage')
$response->dynamic(new Document([
'range' => $range,
'deployments' => $usage[$metrics[0]],
'deploymentsTotal' => $usage[$metrics[0]],
'deploymentsStorage' => $usage[$metrics[1]],
'builds' => $usage[$metrics[2]],
'buildsTotal' => $usage[$metrics[2]],
'buildsStorage' => $usage[$metrics[3]],
'buildsCompute' => $usage[$metrics[4]],
'executions' => $usage[$metrics[5]],
'executionsCompute' => $usage[$metrics[6]],
'buildsTime' => $usage[$metrics[4]],
'executionsTotal' => $usage[$metrics[5]],
'executionsTime' => $usage[$metrics[6]],
]), Response::MODEL_USAGE_FUNCTION);
});
@ -313,14 +313,14 @@ App::get('/v1/functions/usage')
$stats = $usage = [];
$days = $periods[$range];
$metrics = [
'functions',
'deployments',
'deployments.storage',
'builds',
'builds.storage',
'builds.compute',
'executions',
'executions.compute',
METRIC_FUNCTIONS,
METRIC_DEPLOYMENTS,
METRIC_DEPLOYMENTS_STORAGE,
METRIC_BUILDS,
METRIC_BUILDS_STORAGE,
METRIC_BUILDS_COMPUTE,
METRIC_EXECUTIONS,
METRIC_EXECUTIONS_COMPUTE,
];
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
@ -361,14 +361,14 @@ App::get('/v1/functions/usage')
}
$response->dynamic(new Document([
'range' => $range,
'functions' => $usage[$metrics[0]],
'deployments' => $usage[$metrics[1]],
'functionsTotal' => $usage[$metrics[0]],
'deploymentsTotal' => $usage[$metrics[1]],
'deploymentsStorage' => $usage[$metrics[2]],
'builds' => $usage[$metrics[3]],
'buildsTotal' => $usage[$metrics[3]],
'buildsStorage' => $usage[$metrics[4]],
'buildsCompute' => $usage[$metrics[5]],
'executions' => $usage[$metrics[6]],
'executionsCompute' => $usage[$metrics[7]],
'buildsTime' => $usage[$metrics[5]],
'executionsTotal' => $usage[$metrics[6]],
'executionsTime' => $usage[$metrics[7]],
]), Response::MODEL_USAGE_FUNCTIONS);
});

19
app/controllers/shared/api.php
View file

@ -89,7 +89,7 @@ $databaseListener = function (string $event, Document $document, Document $proje
$databaseId = $parts[1] ?? 0;
$queueForUsage
->addMetric(METRIC_COLLECTIONS, $value) // per project
->addMetric(str_replace('databaseId', $databaseId, METRIC_DATABASE_ID_COLLECTIONS), $value) // per database
->addMetric(str_replace('{databaseId}', $databaseId, METRIC_DATABASE_ID_COLLECTIONS), $value) // per database
;
if ($event === Database::EVENT_DOCUMENT_DELETE) {
@ -103,8 +103,8 @@ $databaseListener = function (string $event, Document $document, Document $proje
$collectionId = $parts[3] ?? 0;
$queueForUsage
->addMetric(METRIC_DOCUMENTS, $value) // per project
->addMetric(str_replace('databaseId', $databaseId, METRIC_DATABASE_ID_DOCUMENTS), $value) // per database
->addMetric(str_replace(['databaseId', 'collectionId'], [$databaseId, $collectionId], METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS), $value); // per collection
->addMetric(str_replace('{databaseId}', $databaseId, METRIC_DATABASE_ID_DOCUMENTS), $value) // per database
->addMetric(str_replace(['{databaseId}', '{collectionId}'], [$databaseId, $collectionId], METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS), $value); // per collection
break;
case $document->getCollection() === 'buckets': //buckets
$queueForUsage
@ -116,12 +116,12 @@ $databaseListener = function (string $event, Document $document, Document $proje
break;
case str_starts_with($document->getCollection(), 'bucket_'): // files
$parts = explode('_', $document->getCollection());
$bucketId = $parts[1];
$bucketId = $parts[1];
$queueForUsage
->addMetric(METRIC_FILES, $value) // per project
->addMetric(METRIC_FILES_STORAGE, $document->getAttribute('sizeOriginal') * $value) // per project
->addMetric(str_replace('bucketId', $bucketId, METRIC_BUCKET_ID_FILES), $value) // per bucket
->addMetric(str_replace('bucketId', $bucketId, METRIC_BUCKET_ID_FILES_STORAGE), $document->getAttribute('sizeOriginal') * $value); // per bucket
->addMetric(str_replace('{bucketId}', $bucketId, METRIC_BUCKET_ID_FILES), $value) // per bucket
->addMetric(str_replace('{bucketId}', $bucketId, METRIC_BUCKET_ID_FILES_STORAGE), $document->getAttribute('sizeOriginal') * $value); // per bucket
break;
case $document->getCollection() === 'functions':
$queueForUsage
@ -135,16 +135,15 @@ $databaseListener = function (string $event, Document $document, Document $proje
case $document->getCollection() === 'deployments':
$queueForUsage
->addMetric(METRIC_DEPLOYMENTS, $value) // per project
->addMetric("deployments.storage", $document->getAttribute('size') * $value) // per project
->addMetric(METRIC_DEPLOYMENTS_STORAGE, $document->getAttribute('size') * $value) // per project
->addMetric(str_replace(['resourceType', 'resourceInternalId'], [$document->getAttribute('resourceType'), $document->getAttribute('resourceInternalId')], METRIC_FUNCTION_ID_DEPLOYMENTS), $value)// per function
->addMetric(str_replace(['resourceType', 'resourceInternalId'], [$document->getAttribute('resourceType'), $document->getAttribute('resourceInternalId')], METRIC_FUNCTION_ID_STORAGE), $document->getAttribute('size') * $value);// per function
->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$document->getAttribute('resourceType'), $document->getAttribute('resourceInternalId')], METRIC_FUNCTION_ID_DEPLOYMENTS), $value)// per function
->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$document->getAttribute('resourceType'), $document->getAttribute('resourceInternalId')], METRIC_FUNCTION_ID_STORAGE), $document->getAttribute('size') * $value);// per function
break;
case $document->getCollection() === 'executions':
$queueForUsage
->addMetric(METRIC_EXECUTIONS, $value) // per project
->addMetric(str_replace('functionInternalId', $document->getAttribute('functionInternalId'), METRIC_FUNCTION_ID_EXECUTIONS), $value);// per function
->addMetric(str_replace('{functionInternalId}', $document->getAttribute('functionInternalId'), METRIC_FUNCTION_ID_EXECUTIONS), $value);// per function
break;
default:
break;

10
app/init.php
View file

@ -174,7 +174,7 @@ const APP_AUTH_TYPE_KEY = 'Key';
const APP_AUTH_TYPE_ADMIN = 'Admin';
// Response related
const MAX_OUTPUT_CHUNK_SIZE = 2 * 1024 * 1024; // 2MB
// usage metrics
// Usage metrics
const METRIC_TEAMS = 'teams';
const METRIC_USERS = 'users';
const METRIC_SESSIONS = 'sessions';
@ -192,10 +192,18 @@ const METRIC_BUCKET_ID_FILES_STORAGE = '{bucketId}.files.storage';
const METRIC_FUNCTIONS = 'functions';
const METRIC_DEPLOYMENTS = 'deployments';
const METRIC_DEPLOYMENTS_STORAGE = 'deployments.storage';
const METRIC_BUILDS = 'builds';
const METRIC_BUILDS_STORAGE = 'builds.storage';
const METRIC_BUILDS_COMPUTE = 'builds.compute';
const METRIC_FUNCTION_ID_BUILDS = '{functionInternalId}.builds';
const METRIC_FUNCTION_ID_BUILDS_STORAGE = '{functionInternalId}.builds.storage';
const METRIC_FUNCTION_ID_BUILDS_COMPUTE = '{functionInternalId}.builds.compute';
const METRIC_FUNCTION_ID_DEPLOYMENTS = '{resourceType}.{resourceInternalId}.deployments';
const METRIC_FUNCTION_ID_STORAGE = '{resourceType}.{resourceInternalId}.deployments.storage';
const METRIC_EXECUTIONS = 'executions';
const METRIC_EXECUTIONS_COMPUTE = 'executions.compute';
const METRIC_FUNCTION_ID_EXECUTIONS = '{functionInternalId}.executions';
const METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE = '{functionInternalId}.executions.compute';
$register = new Registry();

12
app/workers/builds.php
View file

@ -254,12 +254,12 @@ class BuildsV1 extends Worker
$this
->getUsageQueue()
->setProject($project)
->addMetric("builds", 1) // per project
->addMetric("builds.storage", $build->getAttribute('size', 0))
->addMetric("builds.compute", $build->getAttribute('duration', 0))
->addMetric("{$function->getInternalId()}" . ".builds", 1) // per function
->addMetric("{$function->getInternalId()}" . ".builds.storage", $build->getAttribute('size', 0))
->addMetric("{$function->getInternalId()}" . ".builds.compute", $build->getAttribute('duration', 0))
->addMetric(METRIC_BUILDS, 1) // per project
->addMetric(METRIC_BUILDS_STORAGE, $build->getAttribute('size', 0))
->addMetric(METRIC_BUILDS_COMPUTE, $build->getAttribute('duration', 0))
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS), 1) // per function
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_STORAGE), $build->getAttribute('size', 0))
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE), $build->getAttribute('duration', 0))
->trigger()
;
}

8
app/workers/functions.php
View file

@ -103,8 +103,8 @@ Server::setResource('execute', function () {
*/
$queueForUsage
->addMetric('executions', 1) // per project
->addMetric("{$function->getId()}" . ".executions", 1); // per function
->addMetric(METRIC_EXECUTIONS, 1) // per project
->addMetric(str_replace('{functionInternalId}', $function->getId(), METRIC_FUNCTION_ID_EXECUTIONS), 1); // per function
}
$execution->setAttribute('status', 'processing');
@ -214,8 +214,8 @@ Server::setResource('execute', function () {
/** Trigger usage queue */
$queueForUsage
->setProject($project)
->addMetric('executions.compute', (int)($execution->getAttribute('duration') * 1000))// per project
->addMetric("{$function->getInternalId()}" . ".executions.compute", (int)($execution->getAttribute('duration') * 1000))// per function
->addMetric(METRIC_EXECUTIONS_COMPUTE, (int)($execution->getAttribute('duration') * 1000))// per project
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE), (int)($execution->getAttribute('duration') * 1000))
->trigger()
;
};

75
docker-compose.yml
View file

@ -85,6 +85,7 @@ services:
- ./public:/usr/src/code/public
- ./src:/usr/src/code/src
- ./dev:/usr/local/dev
depends_on:
- mariadb
- redis
@ -333,7 +334,6 @@ services:
volumes:
- ./app:/usr/src/code/app
- ./src:/usr/src/code/src
#- ./vendor/utopia-php/database:/usr/src/code/vendor/utopia-php/database
depends_on:
- redis
- mariadb
@ -539,43 +539,6 @@ services:
- _APP_LOGGING_PROVIDER
- _APP_LOGGING_CONFIG
appwrite-worker-usage:
entrypoint: worker-usage
<<: *x-logging
container_name: appwrite-worker-usage
image: appwrite-dev
networks:
- appwrite
volumes:
- ./app:/usr/src/code/app
- ./src:/usr/src/code/src
#- ./vendor/utopia-php/database:/usr/src/code/vendor/utopia-php/database
depends_on:
- redis
- mariadb
environment:
- _APP_ENV
- _APP_WORKER_PER_CORE
- _APP_CONNECTIONS_MAX
- _APP_POOL_CLIENTS
- _APP_OPENSSL_KEY_V1
- _APP_DB_HOST
- _APP_DB_PORT
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_CONNECTIONS_DB_CONSOLE
- _APP_CONNECTIONS_DB_PROJECT
- _APP_CONNECTIONS_CACHE
- _APP_CONNECTIONS_QUEUE
- _APP_USAGE_STATS
- DOCKERHUB_PULL_USERNAME
- DOCKERHUB_PULL_PASSWORD
appwrite-maintenance:
entrypoint: maintenance
<<: *x-logging
@ -613,6 +576,42 @@ services:
- _APP_MAINTENANCE_RETENTION_AUDIT
- _APP_MAINTENANCE_RETENTION_SCHEDULES
appwrite-worker-usage:
entrypoint: worker-usage
<<: *x-logging
container_name: appwrite-worker-usage
image: appwrite-dev
networks:
- appwrite
volumes:
- ./app:/usr/src/code/app
- ./src:/usr/src/code/src
depends_on:
- redis
- mariadb
environment:
- _APP_ENV
- _APP_WORKER_PER_CORE
- _APP_CONNECTIONS_MAX
- _APP_POOL_CLIENTS
- _APP_OPENSSL_KEY_V1
- _APP_DB_HOST
- _APP_DB_PORT
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_CONNECTIONS_DB_CONSOLE
- _APP_CONNECTIONS_DB_PROJECT
- _APP_CONNECTIONS_CACHE
- _APP_CONNECTIONS_QUEUE
- _APP_USAGE_STATS
- DOCKERHUB_PULL_USERNAME
- DOCKERHUB_PULL_PASSWORD
appwrite-schedule:
entrypoint: schedule
<<: *x-logging

25
src/Appwrite/Platform/Tasks/Doctor.php
View file

@ -2,20 +2,16 @@
namespace Appwrite\Platform\Tasks;
use Appwrite\URL\URL;
use Utopia\App;
use Utopia\CLI\Console;
use Appwrite\ClamAV\Network;
use Utopia\Logger\Logger;
use Utopia\Queue\Client;
use Utopia\Storage\Device\Local;
use Utopia\Storage\Storage;
use Utopia\Config\Config;
use Utopia\Domains\Domain;
use Utopia\Platform\Action;
use Utopia\Registry\Registry;
use Utopia\DSN\DSN;
use Utopia\Queue;
class Doctor extends Action
{
@ -191,27 +187,6 @@ class Doctor extends Action
Console::error('🔴 ' . str_pad("SMTP", 47, '.') . 'disconnected');
}
try {
$fallbackForRedis = URL::unparse([
'scheme' => 'redis',
'host' => App::getEnv('_APP_REDIS_HOST', 'redis'),
'port' => App::getEnv('_APP_REDIS_PORT', '6379'),
'user' => App::getEnv('_APP_REDIS_USER', ''),
'pass' => App::getEnv('_APP_REDIS_PASS', ''),
]);
$dsn = App::getEnv('_APP_CONNECTIONS_QUEUE', $fallbackForRedis);
$dsn = explode('=', $dsn);
$dsn = $dsn[1] ?? '';
$dsn = new DSN($dsn);
$connection = new Queue\Connection\Redis($dsn->getHost(), $dsn->getPort());
$client = new Client('v1-usage', $connection);
$client->getQueueSize();
Console::success('🟢 ' . str_pad("Usage queue", 50, '.') . 'connected');
} catch (\Throwable $th) {
Console::error('🔴 ' . str_pad("Usage queue", 47, '.') . 'disconnected');
}
\sleep(0.2);
Console::log('');

10
src/Appwrite/Utopia/Response/Model/UsageFunction.php
View file

@ -16,7 +16,7 @@ class UsageFunction extends Model
'default' => '',
'example' => '30d',
])
->addRule('deployments', [
->addRule('deploymentsTotal', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for number of function deployments.',
'default' => [],
@ -30,7 +30,7 @@ class UsageFunction extends Model
'example' => [],
'array' => true
])
->addRule('builds', [
->addRule('buildsTotal', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for number of function builds.',
'default' => [],
@ -44,14 +44,14 @@ class UsageFunction extends Model
'example' => [],
'array' => true
])
->addRule('buildsCompute', [
->addRule('buildsTime', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for function build compute.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('executions', [
->addRule('executionsTotal', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for number of function executions.',
'default' => [],
@ -59,7 +59,7 @@ class UsageFunction extends Model
'array' => true
])
->addRule('executionsCompute', [
->addRule('executionsTime', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for function execution compute.',
'default' => [],

12
src/Appwrite/Utopia/Response/Model/UsageFunctions.php
View file

@ -16,14 +16,14 @@ class UsageFunctions extends Model
'default' => '',
'example' => '30d',
])
->addRule('functions', [
->addRule('functionsTotal', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for number of functions.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('deployments', [
->addRule('deploymentsTotal', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for number of function deployments.',
'default' => [],
@ -37,7 +37,7 @@ class UsageFunctions extends Model
'example' => [],
'array' => true
])
->addRule('builds', [
->addRule('buildsTotal', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for number of function builds.',
'default' => [],
@ -51,14 +51,14 @@ class UsageFunctions extends Model
'example' => [],
'array' => true
])
->addRule('buildsCompute', [
->addRule('buildsTime', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for function build compute.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('executions', [
->addRule('executionsTotal', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for number of function executions.',
'default' => [],
@ -66,7 +66,7 @@ class UsageFunctions extends Model
'array' => true
])
->addRule('executionsCompute', [
->addRule('executionsTime', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for function execution compute.',
'default' => [],