Merge branch 'feat-usage-daemon' into feat-usage-deletes
This commit is contained in:
Damodar Lohani
commit
d779b98365
|
|
@ -14,6 +14,7 @@ use Utopia\Validator\JSON;
|
|||
use Utopia\Database\Database;
|
||||
use Utopia\Database\Document;
|
||||
use Utopia\Database\Query;
|
||||
use Utopia\Database\Adapter\MariaDB;
|
||||
use Utopia\Database\Validator\Key;
|
||||
use Utopia\Database\Validator\Permissions;
|
||||
use Utopia\Database\Validator\QueryValidator;
|
||||
|
|
@ -22,6 +23,7 @@ use Utopia\Database\Validator\Structure;
|
|||
use Utopia\Database\Validator\UID;
|
||||
use Utopia\Database\Exception\Authorization as AuthorizationException;
|
||||
use Utopia\Database\Exception\Duplicate as DuplicateException;
|
||||
use Utopia\Database\Exception\Limit as LimitException;
|
||||
use Utopia\Database\Exception\Structure as StructureException;
|
||||
use Appwrite\Utopia\Response;
|
||||
use Appwrite\Database\Validator\CustomId;
|
||||
|
|
@ -932,6 +934,16 @@ App::post('/v1/database/collections/:collectionId/indexes')
|
|||
throw new Exception('Collection not found', 404);
|
||||
}
|
||||
|
||||
$count = $dbForInternal->count('indexes', [
|
||||
new Query('collectionId', Query::TYPE_EQUAL, [$collectionId])
|
||||
], 61);
|
||||
|
||||
$limit = 64 - MariaDB::getNumberOfDefaultIndexes();
|
||||
|
||||
if ($count >= $limit) {
|
||||
throw new Exception('Index limit exceeded', 400);
|
||||
}
|
||||
|
||||
// Convert Document[] to array of attribute metadata
|
||||
$oldAttributes = \array_map(function ($a) {
|
||||
return $a->getArrayCopy();
|
||||
|
|
@ -956,7 +968,6 @@ App::post('/v1/database/collections/:collectionId/indexes')
|
|||
$lengths[$key] = ($attributeType === Database::VAR_STRING) ? $attributeSize : null;
|
||||
}
|
||||
|
||||
// TODO@kodumbeats should $lengths be a part of the response model?
|
||||
try {
|
||||
$index = $dbForInternal->createDocument('indexes', new Document([
|
||||
'$id' => $collectionId.'_'.$indexId,
|
||||
|
|
@ -969,7 +980,7 @@ App::post('/v1/database/collections/:collectionId/indexes')
|
|||
'orders' => $orders,
|
||||
]));
|
||||
} catch (DuplicateException $th) {
|
||||
throw new Exception('Attribute already exists', 409);
|
||||
throw new Exception('Index already exists', 409);
|
||||
}
|
||||
|
||||
$dbForInternal->purgeDocument('collections', $collectionId);
|
||||
|
|
@ -1387,8 +1398,11 @@ App::patch('/v1/database/collections/:collectionId/documents/:documentId')
|
|||
catch (AuthorizationException $exception) {
|
||||
throw new Exception('Unauthorized permissions', 401);
|
||||
}
|
||||
catch (DuplicateException $exception) {
|
||||
throw new Exception('Document already exists', 409);
|
||||
}
|
||||
catch (StructureException $exception) {
|
||||
throw new Exception('Bad structure. '.$exception->getMessage(), 400);
|
||||
throw new Exception($exception->getMessage(), 400);
|
||||
}
|
||||
|
||||
$usage
|
||||
|
|
@ -1461,4 +1475,4 @@ App::delete('/v1/database/collections/:collectionId/documents/:documentId')
|
|||
;
|
||||
|
||||
$response->noContent();
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -11,15 +11,19 @@ use Utopia\CLI\Console;
|
|||
use Utopia\Database\Adapter\MariaDB;
|
||||
use Utopia\Database\Database;
|
||||
use Utopia\Database\Document;
|
||||
use Utopia\Database\Query;
|
||||
use Utopia\Database\Validator\Authorization;
|
||||
|
||||
/**
|
||||
* Metrics We collect
|
||||
*
|
||||
* General
|
||||
*
|
||||
* requests
|
||||
* network
|
||||
* executions
|
||||
*
|
||||
* Database
|
||||
*
|
||||
* database.collections.create
|
||||
* database.collections.read
|
||||
* database.collections.update
|
||||
|
|
@ -32,10 +36,16 @@ use Utopia\Database\Validator\Authorization;
|
|||
* database.collections.{collectionId}.documents.read
|
||||
* database.collections.{collectionId}.documents.update
|
||||
* database.collections.{collectionId}.documents.delete
|
||||
*
|
||||
* Storage
|
||||
*
|
||||
* storage.buckets.{bucketId}.files.create
|
||||
* storage.buckets.{bucketId}.files.read
|
||||
* storage.buckets.{bucketId}.files.update
|
||||
* storage.buckets.{bucketId}.files.delete
|
||||
*
|
||||
* Users
|
||||
*
|
||||
* users.create
|
||||
* users.read
|
||||
* users.update
|
||||
|
|
@ -71,7 +81,7 @@ $cli
|
|||
Console::title('Usage Aggregation V1');
|
||||
Console::success(APP_NAME . ' usage aggregation process v1 has started');
|
||||
|
||||
$interval = (int) App::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '30'); //30 seconds
|
||||
$interval = (int) App::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '30'); // 30 seconds (by default)
|
||||
$periods = [
|
||||
[
|
||||
'key' => '30m',
|
||||
|
|
@ -189,16 +199,18 @@ $cli
|
|||
],
|
||||
];
|
||||
|
||||
// TODO Maybe move this to the setResource method, and reuse in the http.php file
|
||||
$attempts = 0;
|
||||
$max = 10;
|
||||
$sleep = 1;
|
||||
|
||||
do { // connect to db
|
||||
try {
|
||||
$attempts++;
|
||||
$db = $register->get('db');
|
||||
$redis = $register->get('cache');
|
||||
break; // leave the do-while if successful
|
||||
} catch (\Exception$e) {
|
||||
} catch (\Exception $e) {
|
||||
Console::warning("Database not ready. Retrying connection ({$attempts})...");
|
||||
if ($attempts >= $max) {
|
||||
throw new \Exception('Failed to connect to database: ' . $e->getMessage());
|
||||
|
|
@ -207,6 +219,7 @@ $cli
|
|||
}
|
||||
} while ($attempts < $max);
|
||||
|
||||
// TODO use inject
|
||||
$cacheAdapter = new Cache(new Redis($redis));
|
||||
$dbForProject = new Database(new MariaDB($db), $cacheAdapter);
|
||||
$dbForConsole = new Database(new MariaDB($db), $cacheAdapter);
|
||||
|
|
@ -223,9 +236,13 @@ $cli
|
|||
|
||||
$loopStart = microtime(true);
|
||||
|
||||
/**
|
||||
* Aggregate InfluxDB every 30 seconds
|
||||
*/
|
||||
$client = $register->get('influxdb');
|
||||
if ($client) {
|
||||
$database = $client->selectDB('telegraf');
|
||||
|
||||
// sync data
|
||||
foreach ($globalMetrics as $metric => $options) { //for each metrics
|
||||
foreach ($periods as $period) { // aggregate data for each period
|
||||
|
|
@ -235,10 +252,10 @@ $cli
|
|||
}
|
||||
$end = DateTime::createFromFormat('U', \strtotime('now'))->format(DateTime::RFC3339);
|
||||
|
||||
$table = $options['table']; //which influxdb table to query for this metric
|
||||
$groupBy = empty($options['groupBy']) ? '' : ', "' . $options['groupBy'] . '"'; //some sub level metrics may be grouped by other tags like collectionId, bucketId, etc
|
||||
$table = $options['table']; //Which influxdb table to query for this metric
|
||||
$groupBy = empty($options['groupBy']) ? '' : ', "' . $options['groupBy'] . '"'; //Some sub level metrics may be grouped by other tags like collectionId, bucketId, etc
|
||||
|
||||
$filters = $options['filters'] ?? [];
|
||||
$filters = $options['filters'] ?? []; // Some metrics might have additional filters, like function's status
|
||||
if (!empty($filters)) {
|
||||
$filters = ' AND ' . implode(' AND ', array_map(function ($filter, $value) {
|
||||
return '"' . $filter . '"=\'' . $value . '\'';
|
||||
|
|
@ -246,16 +263,15 @@ $cli
|
|||
}
|
||||
|
||||
$result = $database->query('SELECT sum(value) AS "value" FROM "' . $table . '" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\'' . (empty($filters) ? '' : $filters) . ' GROUP BY time(' . $period['key'] . '), "projectId"' . $groupBy . ' FILL(null)');
|
||||
|
||||
|
||||
$points = $result->getPoints();
|
||||
foreach ($points as $point) {
|
||||
$projectId = $point['projectId'];
|
||||
|
||||
if (!empty($projectId) && $projectId != 'console') {
|
||||
$dbForProject->setNamespace('project_' . $projectId . '_internal');
|
||||
if($metric == 'functions.functionId.executions') {
|
||||
var_dump($points);
|
||||
}
|
||||
$metricUpdated = $metric;
|
||||
|
||||
if (!empty($groupBy)) {
|
||||
$groupedBy = $point[$options['groupBy']] ?? '';
|
||||
if (empty($groupedBy)) {
|
||||
|
|
@ -263,9 +279,11 @@ $cli
|
|||
}
|
||||
$metricUpdated = str_replace($options['groupBy'], $groupedBy, $metric);
|
||||
}
|
||||
|
||||
$time = \strtotime($point['time']);
|
||||
$id = \md5($time . '_' . $period['key'] . '_' . $metricUpdated); //construct unique id for each metric using time, period and metric
|
||||
$id = \md5($time . '_' . $period['key'] . '_' . $metricUpdated); //Construct unique id for each metric using time, period and metric
|
||||
$value = (!empty($point['value'])) ? $point['value'] : 0;
|
||||
|
||||
try {
|
||||
$document = $dbForProject->getDocument('stats', $id);
|
||||
if ($document->isEmpty()) {
|
||||
|
|
@ -279,12 +297,11 @@ $cli
|
|||
]));
|
||||
} else {
|
||||
$dbForProject->updateDocument('stats', $document->getId(),
|
||||
$document->setAttribute('value', $value));
|
||||
$document->setAttribute('value', $value));
|
||||
}
|
||||
$latestTime[$metric][$period['key']] = $time;
|
||||
} catch (\Exception$e) {
|
||||
// if projects are deleted this might fail
|
||||
Console::warning("Failed to save data for project {$projectId} and metric {$metricUpdated}");
|
||||
} catch (\Exception $e) { // if projects are deleted this might fail
|
||||
Console::warning("Failed to save data for project {$projectId} and metric {$metricUpdated}: {$e->getMessage()}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -292,119 +309,245 @@ $cli
|
|||
}
|
||||
}
|
||||
|
||||
if ($iterations % 30 == 0) { //every 15 minutes
|
||||
// aggregate number of objects in database
|
||||
// get count of all the documents per collection -
|
||||
// buckets will have the same
|
||||
/**
|
||||
* Aggregate MariaDB every 15 minutes
|
||||
* Some of the queries here might contain full-table scans.
|
||||
*/
|
||||
if ($iterations % 30 == 0) { // Every 15 minutes aggregate number of objects in database
|
||||
|
||||
$latestProject = null;
|
||||
do {
|
||||
|
||||
do { // Loop over all the projects
|
||||
$projects = $dbForConsole->find('projects', [], 100, orderAfter:$latestProject);
|
||||
if (!empty($projects)) {
|
||||
$latestProject = $projects[array_key_last($projects)];
|
||||
|
||||
foreach ($projects as $project) {
|
||||
$id = $project->getId();
|
||||
if (empty($projects)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// get total storage
|
||||
$dbForProject->setNamespace('project_' . $id . '_internal');
|
||||
$storageTotal = $dbForProject->sum('files', 'sizeOriginal') + $dbForProject->sum('tags', 'size');
|
||||
$latestProject = $projects[array_key_last($projects)];
|
||||
|
||||
foreach ($projects as $project) {
|
||||
$projectId = $project->getId();
|
||||
|
||||
// Get total storage
|
||||
$dbForProject->setNamespace('project_' . $projectId . '_internal');
|
||||
$storageTotal = $dbForProject->sum('files', 'sizeOriginal') + $dbForProject->sum('tags', 'size');
|
||||
|
||||
$time = (int) (floor(time() / 1800) * 1800); // Time rounded to nearest 30 minutes
|
||||
$id = \md5($time . '_30m_storage.total'); //Construct unique id for each metric using time, period and metric
|
||||
$document = $dbForProject->getDocument('stats', $id);
|
||||
if ($document->isEmpty()) {
|
||||
$dbForProject->createDocument('stats', new Document([
|
||||
'$id' => $dbForProject->getId(),
|
||||
'period' => '15m',
|
||||
'time' => time(),
|
||||
'$id' => $id,
|
||||
'period' => '30m',
|
||||
'time' => $time,
|
||||
'metric' => 'storage.total',
|
||||
'value' => $storageTotal,
|
||||
'type' => 1,
|
||||
]));
|
||||
} else {
|
||||
$dbForProject->updateDocument('stats', $document->getId(),
|
||||
$document->setAttribute('value', $storageTotal));
|
||||
}
|
||||
|
||||
$time = (int) (floor(time() / 86400) * 86400); // Time rounded to nearest day
|
||||
$id = \md5($time . '_1d_storage.total'); //Construct unique id for each metric using time, period and metric
|
||||
$document = $dbForProject->getDocument('stats', $id);
|
||||
if ($document->isEmpty()) {
|
||||
$dbForProject->createDocument('stats', new Document([
|
||||
'$id' => $id,
|
||||
'period' => '1d',
|
||||
'time' => $time,
|
||||
'metric' => 'storage.total',
|
||||
'value' => $storageTotal,
|
||||
'type' => 1,
|
||||
]));
|
||||
} else {
|
||||
$dbForProject->updateDocument('stats', $document->getId(),
|
||||
$document->setAttribute('value', $storageTotal));
|
||||
}
|
||||
|
||||
$collections = [
|
||||
'users' => [
|
||||
'namespace' => 'internal',
|
||||
],
|
||||
'collections' => [
|
||||
'metricPrefix' => 'database',
|
||||
'namespace' => 'internal',
|
||||
'subCollections' => [
|
||||
'documents' => [
|
||||
'namespace' => 'external',
|
||||
],
|
||||
$collections = [
|
||||
'users' => [
|
||||
'namespace' => 'internal',
|
||||
],
|
||||
'collections' => [
|
||||
'metricPrefix' => 'database',
|
||||
'namespace' => 'internal',
|
||||
'subCollections' => [ // Some collections, like collections and later buckets have child collections that need counting
|
||||
'documents' => [
|
||||
'namespace' => 'external',
|
||||
],
|
||||
],
|
||||
'files' => [
|
||||
'metricPrefix' => 'storage',
|
||||
'namespace' => 'internal',
|
||||
],
|
||||
];
|
||||
foreach ($collections as $collection => $options) {
|
||||
try {
|
||||
$dbForProject->setNamespace("project_{$id}_{$options['namespace']}");
|
||||
$count = $dbForProject->count($collection);
|
||||
$dbForProject->setNamespace("project_{$id}_internal");
|
||||
$metricPrefix = $options['metricPrefix'] ?? '';
|
||||
$metric = empty($metricPrefix) ? "{$collection}.count" : "{$metricPrefix}.{$collection}.count";
|
||||
],
|
||||
'files' => [
|
||||
'metricPrefix' => 'storage',
|
||||
'namespace' => 'internal',
|
||||
],
|
||||
];
|
||||
|
||||
foreach ($collections as $collection => $options) {
|
||||
try {
|
||||
$dbForProject->setNamespace("project_{$projectId}_{$options['namespace']}");
|
||||
$count = $dbForProject->count($collection);
|
||||
$dbForProject->setNamespace("project_{$projectId}_internal");
|
||||
$metricPrefix = $options['metricPrefix'] ?? '';
|
||||
$metric = empty($metricPrefix) ? "{$collection}.count" : "{$metricPrefix}.{$collection}.count";
|
||||
|
||||
$time = (int) (floor(time() / 1800) * 1800); // Time rounded to nearest 30 minutes
|
||||
$id = \md5($time . '_30m_' . $metric); //Construct unique id for each metric using time, period and metric
|
||||
$document = $dbForProject->getDocument('stats', $id);
|
||||
if ($document->isEmpty()) {
|
||||
$dbForProject->createDocument('stats', new Document([
|
||||
'$id' => $dbForProject->getId(),
|
||||
'time' => time(),
|
||||
'period' => '15m',
|
||||
'$id' => $id,
|
||||
'time' => $time,
|
||||
'period' => '30m',
|
||||
'metric' => $metric,
|
||||
'value' => $count,
|
||||
'type' => 1,
|
||||
]));
|
||||
} else {
|
||||
$dbForProject->updateDocument('stats', $document->getId(),
|
||||
$document->setAttribute('value', $count));
|
||||
}
|
||||
|
||||
$subCollections = $options['subCollections'] ?? [];
|
||||
if (!empty($subCollections)) {
|
||||
$latestParent = null;
|
||||
$subCollectionCounts = []; //total project level count of sub collections
|
||||
do {
|
||||
$dbForProject->setNamespace("project_{$id}_{$options['namespace']}");
|
||||
$parents = $dbForProject->find($collection, [], 100, orderAfter:$latestParent);
|
||||
if (!empty($parents)) {
|
||||
$latestParent = $parents[array_key_last($parents)];
|
||||
foreach ($parents as $parent) {
|
||||
foreach ($subCollections as $subCollection => $subOptions) {
|
||||
$dbForProject->setNamespace("project_{$id}_{$subOptions['namespace']}");
|
||||
$count = $dbForProject->count($parent->getId());
|
||||
$subCollectionsCounts[$subCollection] = ($subCollectionCounts[$subCollection] ?? 0) + $count;
|
||||
$time = (int) (floor(time() / 86400) * 86400); // Time rounded to nearest day
|
||||
$id = \md5($time . '_1d_' . $metric); //Construct unique id for each metric using time, period and metric
|
||||
$document = $dbForProject->getDocument('stats', $id);
|
||||
if ($document->isEmpty()) {
|
||||
$dbForProject->createDocument('stats', new Document([
|
||||
'$id' => $id,
|
||||
'time' => $time,
|
||||
'period' => '1d',
|
||||
'metric' => $metric,
|
||||
'value' => $count,
|
||||
'type' => 1,
|
||||
]));
|
||||
} else {
|
||||
$dbForProject->updateDocument('stats', $document->getId(),
|
||||
$document->setAttribute('value', $count));
|
||||
}
|
||||
|
||||
$dbForProject->setNamespace("project_{$id}_internal");
|
||||
$dbForProject->createDocument('stats', new Document([
|
||||
'$id' => $dbForProject->getId(),
|
||||
'time' => time(),
|
||||
'period' => '15m',
|
||||
'metric' => empty($metricPrefix) ? "{$collection}.{$parent->getId()}.{$subCollection}.count" : "{$metricPrefix}.{$collection}.{$parent->getId()}.{$subCollection}.count",
|
||||
'value' => $count,
|
||||
'type' => 1,
|
||||
]));
|
||||
}
|
||||
}
|
||||
$subCollections = $options['subCollections'] ?? [];
|
||||
|
||||
if (empty($subCollections)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$latestParent = null;
|
||||
$subCollectionCounts = []; //total project level count of sub collections
|
||||
|
||||
do { // Loop over all the parent collection document for each sub collection
|
||||
$dbForProject->setNamespace("project_{$projectId}_{$options['namespace']}");
|
||||
$parents = $dbForProject->find($collection, [], 100, orderAfter:$latestParent); // Get all the parents for the sub collections for example for documents, this will get all the collections
|
||||
|
||||
if (empty($parents)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$latestParent = $parents[array_key_last($parents)];
|
||||
|
||||
foreach ($parents as $parent) {
|
||||
foreach ($subCollections as $subCollection => $subOptions) { // Sub collection counts, like database.collections.collectionId.documents.count
|
||||
$dbForProject->setNamespace("project_{$projectId}_{$subOptions['namespace']}");
|
||||
$count = $dbForProject->count($parent->getId());
|
||||
|
||||
$subCollectionCounts[$subCollection] = ($subCollectionCounts[$subCollection] ?? 0) + $count; // Project level counts for sub collections like database.documents.count
|
||||
|
||||
$dbForProject->setNamespace("project_{$projectId}_internal");
|
||||
|
||||
$metric = empty($metricPrefix) ? "{$collection}.{$parent->getId()}.{$subCollection}.count" : "{$metricPrefix}.{$collection}.{$parent->getId()}.{$subCollection}.count";
|
||||
$time = (int) (floor(time() / 1800) * 1800); // Time rounded to nearest 30 minutes
|
||||
$id = \md5($time . '_30m_' . $metric); //Construct unique id for each metric using time, period and metric
|
||||
$document = $dbForProject->getDocument('stats', $id);
|
||||
if ($document->isEmpty()) {
|
||||
$dbForProject->createDocument('stats', new Document([
|
||||
'$id' => $id,
|
||||
'time' => $time,
|
||||
'period' => '30m',
|
||||
'metric' => $metric,
|
||||
'value' => $count,
|
||||
'type' => 1,
|
||||
]));
|
||||
} else {
|
||||
$dbForProject->updateDocument('stats', $document->getId(),
|
||||
$document->setAttribute('value', $count));
|
||||
}
|
||||
} while (!empty($parents));
|
||||
|
||||
foreach ($subCollectionsCounts as $subCollection => $count) {
|
||||
$dbForProject->setNamespace("project_{$id}_internal");
|
||||
$dbForProject->createDocument('stats', new Document([
|
||||
'$id' => $dbForProject->getId(),
|
||||
'time' => time(),
|
||||
'period' => '15m',
|
||||
'metric' => empty($metricPrefix) ? "{$subCollection}.count" : "{$metricPrefix}.{$subCollection}.count",
|
||||
'value' => $count,
|
||||
'type' => 1,
|
||||
]));
|
||||
$time = (int) (floor(time() / 86400) * 86400); // Time rounded to nearest day
|
||||
$id = \md5($time . '_1d_' . $metric); //Construct unique id for each metric using time, period and metric
|
||||
$document = $dbForProject->getDocument('stats', $id);
|
||||
if ($document->isEmpty()) {
|
||||
$dbForProject->createDocument('stats', new Document([
|
||||
'$id' => $id,
|
||||
'time' => $time,
|
||||
'period' => '1d',
|
||||
'metric' => $metric,
|
||||
'value' => $count,
|
||||
'type' => 1,
|
||||
]));
|
||||
} else {
|
||||
$dbForProject->updateDocument('stats', $document->getId(),
|
||||
$document->setAttribute('value', $count));
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (\Exception$e) {
|
||||
Console::warning("Failed to save database counters data for project {$collection}");
|
||||
} while (!empty($parents));
|
||||
|
||||
/**
|
||||
* Inserting project level counts for sub collections like database.documents.count
|
||||
*/
|
||||
foreach ($subCollectionCounts as $subCollection => $count) {
|
||||
$dbForProject->setNamespace("project_{$projectId}_internal");
|
||||
|
||||
$metric = empty($metricPrefix) ? "{$subCollection}.count" : "{$metricPrefix}.{$subCollection}.count";
|
||||
|
||||
$time = (int) (floor(time() / 1800) * 1800); // Time rounded to nearest 30 minutes
|
||||
$id = \md5($time . '_30m_' . $metric); //Construct unique id for each metric using time, period and metric
|
||||
$document = $dbForProject->getDocument('stats', $id);
|
||||
if ($document->isEmpty()) {
|
||||
$dbForProject->createDocument('stats', new Document([
|
||||
'$id' => $id,
|
||||
'time' => $time,
|
||||
'period' => '30m',
|
||||
'metric' => $metric,
|
||||
'value' => $count,
|
||||
'type' => 1,
|
||||
]));
|
||||
} else {
|
||||
$dbForProject->updateDocument('stats', $document->getId(),
|
||||
$document->setAttribute('value', $count));
|
||||
}
|
||||
|
||||
$time = (int) (floor(time() / 86400) * 86400); // Time rounded to nearest day
|
||||
$id = \md5($time . '_1d_' . $metric); //Construct unique id for each metric using time, period and metric
|
||||
$document = $dbForProject->getDocument('stats', $id);
|
||||
if ($document->isEmpty()) {
|
||||
$dbForProject->createDocument('stats', new Document([
|
||||
'$id' => $id,
|
||||
'time' => $time,
|
||||
'period' => '1d',
|
||||
'metric' => $metric,
|
||||
'value' => $count,
|
||||
'type' => 1,
|
||||
]));
|
||||
} else {
|
||||
$dbForProject->updateDocument('stats', $document->getId(),
|
||||
$document->setAttribute('value', $count));
|
||||
}
|
||||
}
|
||||
} catch (\Exception$e) {
|
||||
Console::warning("Failed to save database counters data for project {$collection}: {$e->getMessage()}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} while (!empty($projects));
|
||||
}
|
||||
|
||||
$iterations++;
|
||||
$loopTook = microtime(true) - $loopStart;
|
||||
$now = date('d-m-Y H:i:s', time());
|
||||
|
||||
Console::info("[{$now}] Aggregation took {$loopTook} seconds");
|
||||
}, $interval);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -423,13 +423,13 @@ $auth = $this->getParam('auth', []);
|
|||
data-scope="console">
|
||||
<ul class="tiles cell-3 margin-bottom-small">
|
||||
<?php foreach ($providers as $provider => $data):
|
||||
if (isset($data['enabled']) && !$data['enabled']) {continue;}
|
||||
if (isset($data['mock']) && $data['mock']) {continue;}
|
||||
$sandbox = $data['sandbox'] ?? false;
|
||||
$form = $data['form'] ?? false;
|
||||
$name = $data['name'] ?? 'Unknown';
|
||||
$beta = $data['beta'] ?? false;
|
||||
?>
|
||||
if (isset($data['enabled']) && !$data['enabled']) {continue;}
|
||||
// if (isset($data['mock']) && $data['mock']) {continue;}
|
||||
$sandbox = $data['sandbox'] ?? false;
|
||||
$form = $data['form'] ?? false;
|
||||
$name = $data['name'] ?? 'Unknown';
|
||||
$beta = $data['beta'] ?? false;
|
||||
?>
|
||||
<li class="<?php echo (isset($data['enabled']) && !$data['enabled']) ? 'dev-feature' : ''; ?>">
|
||||
<div data-ui-modal class="modal close" data-button-alias="none" data-open-event="provider-update-<?php echo $provider; ?>">
|
||||
<button type="button" class="close pull-end" data-ui-modal-close=""><i class="icon-cancel"></i></button>
|
||||
|
|
|
|||
|
|
@ -18,8 +18,14 @@ Console::success(APP_NAME.' deletes worker v1 has started'."\n");
|
|||
|
||||
class DeletesV1 extends Worker
|
||||
{
|
||||
/**
|
||||
* @var array
|
||||
*/
|
||||
public $args = [];
|
||||
|
||||
/**
|
||||
* @var Database
|
||||
*/
|
||||
protected $consoleDB = null;
|
||||
|
||||
public function init(): void
|
||||
|
|
@ -33,7 +39,7 @@ class DeletesV1 extends Worker
|
|||
|
||||
switch (strval($type)) {
|
||||
case DELETE_TYPE_DOCUMENT:
|
||||
$document = $this->args['document'] ?? '';
|
||||
$document = $this->args['document'] ?? [];
|
||||
$document = new Document($document);
|
||||
|
||||
switch ($document->getCollection()) {
|
||||
|
|
@ -113,7 +119,8 @@ class DeletesV1 extends Worker
|
|||
* @param Document $document teams document
|
||||
* @param string $projectId
|
||||
*/
|
||||
protected function deleteMemberships(Document $document, $projectId) {
|
||||
protected function deleteMemberships(Document $document, string $projectId): void
|
||||
{
|
||||
$teamId = $document->getAttribute('teamId', '');
|
||||
|
||||
// Delete Memberships
|
||||
|
|
@ -125,7 +132,7 @@ class DeletesV1 extends Worker
|
|||
/**
|
||||
* @param Document $document project document
|
||||
*/
|
||||
protected function deleteProject(Document $document)
|
||||
protected function deleteProject(Document $document): void
|
||||
{
|
||||
$projectId = $document->getId();
|
||||
// Delete all DBs
|
||||
|
|
@ -144,7 +151,7 @@ class DeletesV1 extends Worker
|
|||
* @param Document $document user document
|
||||
* @param string $projectId
|
||||
*/
|
||||
protected function deleteUser(Document $document, $projectId)
|
||||
protected function deleteUser(Document $document, string $projectId): void
|
||||
{
|
||||
$userId = $document->getId();
|
||||
|
||||
|
|
@ -169,9 +176,9 @@ class DeletesV1 extends Worker
|
|||
/**
|
||||
* @param int $timestamp
|
||||
*/
|
||||
protected function deleteExecutionLogs($timestamp)
|
||||
protected function deleteExecutionLogs(int $timestamp): void
|
||||
{
|
||||
$this->deleteForProjectIds(function($projectId) use ($timestamp) {
|
||||
$this->deleteForProjectIds(function(string $projectId) use ($timestamp) {
|
||||
if (!($dbForInternal = $this->getInternalDB($projectId))) {
|
||||
throw new Exception('Failed to get projectDB for project '.$projectId);
|
||||
}
|
||||
|
|
@ -186,7 +193,7 @@ class DeletesV1 extends Worker
|
|||
/**
|
||||
* @param int $timestamp
|
||||
*/
|
||||
protected function deleteAbuseLogs($timestamp)
|
||||
protected function deleteAbuseLogs(int $timestamp): void
|
||||
{
|
||||
if($timestamp == 0) {
|
||||
throw new Exception('Failed to delete audit logs. No timestamp provided');
|
||||
|
|
@ -206,7 +213,7 @@ class DeletesV1 extends Worker
|
|||
/**
|
||||
* @param int $timestamp
|
||||
*/
|
||||
protected function deleteAuditLogs($timestamp)
|
||||
protected function deleteAuditLogs(int $timestamp): void
|
||||
{
|
||||
if($timestamp == 0) {
|
||||
throw new Exception('Failed to delete audit logs. No timestamp provided');
|
||||
|
|
@ -224,7 +231,7 @@ class DeletesV1 extends Worker
|
|||
* @param Document $document function document
|
||||
* @param string $projectId
|
||||
*/
|
||||
protected function deleteFunction(Document $document, $projectId)
|
||||
protected function deleteFunction(Document $document, string $projectId): void
|
||||
{
|
||||
$dbForInternal = $this->getInternalDB($projectId);
|
||||
$device = new Local(APP_STORAGE_FUNCTIONS.'/app-'.$projectId);
|
||||
|
|
@ -281,7 +288,7 @@ class DeletesV1 extends Worker
|
|||
/**
|
||||
* @param callable $callback
|
||||
*/
|
||||
protected function deleteForProjectIds(callable $callback)
|
||||
protected function deleteForProjectIds(callable $callback): void
|
||||
{
|
||||
$count = 0;
|
||||
$chunk = 0;
|
||||
|
|
@ -292,12 +299,12 @@ class DeletesV1 extends Worker
|
|||
$executionStart = \microtime(true);
|
||||
|
||||
while($sum === $limit) {
|
||||
$chunk++;
|
||||
|
||||
Authorization::disable();
|
||||
$projects = $this->getConsoleDB()->find('projects', [], $limit);
|
||||
$projects = $this->getConsoleDB()->find('projects', [], $limit, ($chunk * $limit));
|
||||
Authorization::reset();
|
||||
|
||||
$chunk++;
|
||||
|
||||
$projectIds = array_map (function ($project) {
|
||||
return $project->getId();
|
||||
}, $projects);
|
||||
|
|
@ -321,7 +328,7 @@ class DeletesV1 extends Worker
|
|||
* @param Database $database
|
||||
* @param callable $callback
|
||||
*/
|
||||
protected function deleteByGroup(string $collection, array $queries, Database $database, callable $callback = null)
|
||||
protected function deleteByGroup(string $collection, array $queries, Database $database, callable $callback = null): void
|
||||
{
|
||||
$count = 0;
|
||||
$chunk = 0;
|
||||
|
|
@ -357,9 +364,8 @@ class DeletesV1 extends Worker
|
|||
|
||||
/**
|
||||
* @param Document $document certificates document
|
||||
* @return Database
|
||||
*/
|
||||
protected function deleteCertificates(Document $document)
|
||||
protected function deleteCertificates(Document $document): void
|
||||
{
|
||||
$domain = $document->getAttribute('domain');
|
||||
$directory = APP_STORAGE_CERTIFICATES . '/' . $domain;
|
||||
|
|
|
|||
|
|
@ -63,7 +63,12 @@
|
|||
"adhocore/jwt": "1.1.2",
|
||||
"slickdeals/statsd": "3.1.0"
|
||||
},
|
||||
"repositories": [],
|
||||
"repositories": [
|
||||
{
|
||||
"type": "git",
|
||||
"url": "https://github.com/utopia-php/database"
|
||||
}
|
||||
],
|
||||
"require-dev": {
|
||||
"appwrite/sdk-generator": "0.13.0",
|
||||
"swoole/ide-helper": "4.6.7",
|
||||
|
|
|
|||
22
composer.lock
generated
22
composer.lock
generated
|
|
@ -4,7 +4,7 @@
|
|||
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
|
||||
"This file is @generated automatically"
|
||||
],
|
||||
"content-hash": "aac413429914bd9299a7ae722f00cef8",
|
||||
"content-hash": "175f077512c575216c4c88f1d33c6d00",
|
||||
"packages": [
|
||||
{
|
||||
"name": "adhocore/jwt",
|
||||
|
|
@ -1987,15 +1987,9 @@
|
|||
"version": "dev-feat-adjusted-query-validator",
|
||||
"source": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/utopia-php/database.git",
|
||||
"url": "https://github.com/utopia-php/database",
|
||||
"reference": "cb73391371f70ddb54bc0000064b15c5f173cb7a"
|
||||
},
|
||||
"dist": {
|
||||
"type": "zip",
|
||||
"url": "https://api.github.com/repos/utopia-php/database/zipball/cb73391371f70ddb54bc0000064b15c5f173cb7a",
|
||||
"reference": "cb73391371f70ddb54bc0000064b15c5f173cb7a",
|
||||
"shasum": ""
|
||||
},
|
||||
"require": {
|
||||
"ext-mongodb": "*",
|
||||
"ext-pdo": "*",
|
||||
|
|
@ -2017,7 +2011,11 @@
|
|||
"Utopia\\Database\\": "src/Database"
|
||||
}
|
||||
},
|
||||
"notification-url": "https://packagist.org/downloads/",
|
||||
"autoload-dev": {
|
||||
"psr-4": {
|
||||
"Utopia\\Tests\\": "tests/Database"
|
||||
}
|
||||
},
|
||||
"license": [
|
||||
"MIT"
|
||||
],
|
||||
|
|
@ -2039,10 +2037,6 @@
|
|||
"upf",
|
||||
"utopia"
|
||||
],
|
||||
"support": {
|
||||
"issues": "https://github.com/utopia-php/database/issues",
|
||||
"source": "https://github.com/utopia-php/database/tree/feat-adjusted-query-validator"
|
||||
},
|
||||
"time": "2021-08-23T14:18:47+00:00"
|
||||
},
|
||||
{
|
||||
|
|
@ -6287,5 +6281,5 @@
|
|||
"platform-overrides": {
|
||||
"php": "8.0"
|
||||
},
|
||||
"plugin-api-version": "2.0.0"
|
||||
"plugin-api-version": "2.1.0"
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1089,4 +1089,86 @@ trait DatabaseBase
|
|||
|
||||
return $data;
|
||||
}
|
||||
|
||||
/**
|
||||
* @depends testDefaultPermissions
|
||||
*/
|
||||
public function testUniqueIndexDuplicate(array $data): array
|
||||
{
|
||||
$uniqueIndex = $this->client->call(Client::METHOD_POST, '/database/collections/' . $data['moviesId'] . '/indexes', array_merge([
|
||||
'content-type' => 'application/json',
|
||||
'x-appwrite-project' => $this->getProject()['$id'],
|
||||
'x-appwrite-key' => $this->getProject()['apiKey']
|
||||
]), [
|
||||
'indexId' => 'unique_title',
|
||||
'type' => 'unique',
|
||||
'attributes' => ['title'],
|
||||
]);
|
||||
|
||||
$this->assertEquals($uniqueIndex['headers']['status-code'], 201);
|
||||
|
||||
sleep(2);
|
||||
|
||||
// test for failure
|
||||
$duplicate = $this->client->call(Client::METHOD_POST, '/database/collections/' . $data['moviesId'] . '/documents', array_merge([
|
||||
'content-type' => 'application/json',
|
||||
'x-appwrite-project' => $this->getProject()['$id'],
|
||||
], $this->getHeaders()), [
|
||||
'documentId' => 'unique()',
|
||||
'data' => [
|
||||
'title' => 'Captain America',
|
||||
'releaseYear' => 1944,
|
||||
'actors' => [
|
||||
'Chris Evans',
|
||||
'Samuel Jackson',
|
||||
]
|
||||
],
|
||||
'read' => ['user:'.$this->getUser()['$id']],
|
||||
'write' => ['user:'.$this->getUser()['$id']],
|
||||
]);
|
||||
|
||||
$this->assertEquals(409, $duplicate['headers']['status-code']);
|
||||
|
||||
// Test for exception when updating document to conflict
|
||||
$document = $this->client->call(Client::METHOD_POST, '/database/collections/' . $data['moviesId'] . '/documents', array_merge([
|
||||
'content-type' => 'application/json',
|
||||
'x-appwrite-project' => $this->getProject()['$id'],
|
||||
], $this->getHeaders()), [
|
||||
'documentId' => 'unique()',
|
||||
'data' => [
|
||||
'title' => 'Captain America 5',
|
||||
'releaseYear' => 1944,
|
||||
'actors' => [
|
||||
'Chris Evans',
|
||||
'Samuel Jackson',
|
||||
]
|
||||
],
|
||||
'read' => ['user:'.$this->getUser()['$id']],
|
||||
'write' => ['user:'.$this->getUser()['$id']],
|
||||
]);
|
||||
|
||||
$this->assertEquals(201, $document['headers']['status-code']);
|
||||
|
||||
// Test for exception when updating document to conflict
|
||||
$duplicate = $this->client->call(Client::METHOD_PATCH, '/database/collections/' . $data['moviesId'] . '/documents/' . $document['body']['$id'], array_merge([
|
||||
'content-type' => 'application/json',
|
||||
'x-appwrite-project' => $this->getProject()['$id'],
|
||||
], $this->getHeaders()), [
|
||||
'documentId' => 'unique()',
|
||||
'data' => [
|
||||
'title' => 'Captain America',
|
||||
'releaseYear' => 1944,
|
||||
'actors' => [
|
||||
'Chris Evans',
|
||||
'Samuel Jackson',
|
||||
]
|
||||
],
|
||||
'read' => ['user:'.$this->getUser()['$id']],
|
||||
'write' => ['user:'.$this->getUser()['$id']],
|
||||
]);
|
||||
|
||||
$this->assertEquals(409, $duplicate['headers']['status-code']);
|
||||
|
||||
return $data;
|
||||
}
|
||||
}
|
||||
|
|
@ -306,4 +306,102 @@ class DatabaseCustomServerTest extends Scope
|
|||
|
||||
$this->assertEquals($response['headers']['status-code'], 404);
|
||||
}
|
||||
|
||||
public function testIndexLimitException()
|
||||
{
|
||||
$collection = $this->client->call(Client::METHOD_POST, '/database/collections', array_merge([
|
||||
'content-type' => 'application/json',
|
||||
'x-appwrite-project' => $this->getProject()['$id'],
|
||||
'x-appwrite-key' => $this->getProject()['apiKey']
|
||||
]), [
|
||||
'collectionId' => 'testLimitException',
|
||||
'name' => 'testLimitException',
|
||||
'read' => ['role:all'],
|
||||
'write' => ['role:all'],
|
||||
'permission' => 'document',
|
||||
]);
|
||||
|
||||
$this->assertEquals($collection['headers']['status-code'], 201);
|
||||
$this->assertEquals($collection['body']['name'], 'testLimitException');
|
||||
|
||||
$collectionId = $collection['body']['$id'];
|
||||
|
||||
// add unique attributes for indexing
|
||||
for ($i=0; $i < 64; $i++) {
|
||||
// $this->assertEquals(true, static::getDatabase()->createAttribute('indexLimit', "test{$i}", Database::VAR_STRING, 16, true));
|
||||
$attribute = $this->client->call(Client::METHOD_POST, '/database/collections/' . $collectionId . '/attributes/string', array_merge([
|
||||
'content-type' => 'application/json',
|
||||
'x-appwrite-project' => $this->getProject()['$id'],
|
||||
'x-appwrite-key' => $this->getProject()['apiKey']
|
||||
]), [
|
||||
'attributeId' => "attribute{$i}",
|
||||
'size' => 64,
|
||||
'required' => true,
|
||||
]);
|
||||
|
||||
$this->assertEquals($attribute['headers']['status-code'], 201);
|
||||
}
|
||||
|
||||
sleep(5);
|
||||
|
||||
$collection = $this->client->call(Client::METHOD_GET, '/database/collections/' . $collectionId, array_merge([
|
||||
'content-type' => 'application/json',
|
||||
'x-appwrite-project' => $this->getProject()['$id'],
|
||||
'x-appwrite-key' => $this->getProject()['apiKey']
|
||||
]));
|
||||
|
||||
$this->assertEquals($collection['headers']['status-code'], 200);
|
||||
$this->assertEquals($collection['body']['name'], 'testLimitException');
|
||||
$this->assertIsArray($collection['body']['attributes']);
|
||||
$this->assertIsArray($collection['body']['indexes']);
|
||||
$this->assertCount(64, $collection['body']['attributes']);
|
||||
$this->assertCount(0, $collection['body']['indexes']);
|
||||
|
||||
// testing for indexLimit = 64
|
||||
// MariaDB, MySQL, and MongoDB create 3 indexes per new collection
|
||||
// Add up to the limit, then check if the next index throws IndexLimitException
|
||||
for ($i=0; $i < 61; $i++) {
|
||||
// $this->assertEquals(true, static::getDatabase()->createIndex('indexLimit', "index{$i}", Database::INDEX_KEY, ["test{$i}"], [16]));
|
||||
$index = $this->client->call(Client::METHOD_POST, '/database/collections/' . $collectionId . '/indexes', array_merge([
|
||||
'content-type' => 'application/json',
|
||||
'x-appwrite-project' => $this->getProject()['$id'],
|
||||
'x-appwrite-key' => $this->getProject()['apiKey']
|
||||
]), [
|
||||
'indexId' => "key_attribute{$i}",
|
||||
'type' => 'key',
|
||||
'attributes' => ["attribute{$i}"],
|
||||
]);
|
||||
|
||||
$this->assertEquals(201, $index['headers']['status-code']);
|
||||
$this->assertEquals("key_attribute{$i}", $index['body']['key']);
|
||||
}
|
||||
|
||||
sleep(5);
|
||||
|
||||
$collection = $this->client->call(Client::METHOD_GET, '/database/collections/' . $collectionId, array_merge([
|
||||
'content-type' => 'application/json',
|
||||
'x-appwrite-project' => $this->getProject()['$id'],
|
||||
'x-appwrite-key' => $this->getProject()['apiKey']
|
||||
]));
|
||||
|
||||
$this->assertEquals($collection['headers']['status-code'], 200);
|
||||
$this->assertEquals($collection['body']['name'], 'testLimitException');
|
||||
$this->assertIsArray($collection['body']['attributes']);
|
||||
$this->assertIsArray($collection['body']['indexes']);
|
||||
$this->assertCount(64, $collection['body']['attributes']);
|
||||
$this->assertCount(61, $collection['body']['indexes']);
|
||||
|
||||
$tooMany = $this->client->call(Client::METHOD_POST, '/database/collections/' . $collectionId . '/indexes', array_merge([
|
||||
'content-type' => 'application/json',
|
||||
'x-appwrite-project' => $this->getProject()['$id'],
|
||||
'x-appwrite-key' => $this->getProject()['apiKey']
|
||||
]), [
|
||||
'indexId' => 'tooMany',
|
||||
'type' => 'key',
|
||||
'attributes' => ['attribute61'],
|
||||
]);
|
||||
|
||||
$this->assertEquals(400, $tooMany['headers']['status-code']);
|
||||
$this->assertEquals('Index limit exceeded', $tooMany['body']['message']);
|
||||
}
|
||||
}
|
||||
Loading…
Reference in a new issue