fix: stats migration
This commit is contained in:
parent
704fd8693c
commit
75e80a210c
2 changed files with 137 additions and 5 deletions
8
composer.lock
generated
8
composer.lock
generated
|
@ -2064,12 +2064,12 @@
|
||||||
"source": {
|
"source": {
|
||||||
"type": "git",
|
"type": "git",
|
||||||
"url": "https://github.com/utopia-php/database.git",
|
"url": "https://github.com/utopia-php/database.git",
|
||||||
"reference": "4feca26ef1535751e7098653fe3f4a64306b2d78"
|
"reference": "9b724a682c8c0e51f272f73d1941a8f3d4b52a01"
|
||||||
},
|
},
|
||||||
"dist": {
|
"dist": {
|
||||||
"type": "zip",
|
"type": "zip",
|
||||||
"url": "https://api.github.com/repos/utopia-php/database/zipball/4feca26ef1535751e7098653fe3f4a64306b2d78",
|
"url": "https://api.github.com/repos/utopia-php/database/zipball/9b724a682c8c0e51f272f73d1941a8f3d4b52a01",
|
||||||
"reference": "4feca26ef1535751e7098653fe3f4a64306b2d78",
|
"reference": "9b724a682c8c0e51f272f73d1941a8f3d4b52a01",
|
||||||
"shasum": ""
|
"shasum": ""
|
||||||
},
|
},
|
||||||
"require": {
|
"require": {
|
||||||
|
@ -2120,7 +2120,7 @@
|
||||||
"issues": "https://github.com/utopia-php/database/issues",
|
"issues": "https://github.com/utopia-php/database/issues",
|
||||||
"source": "https://github.com/utopia-php/database/tree/fix-update-attribute-datetime"
|
"source": "https://github.com/utopia-php/database/tree/fix-update-attribute-datetime"
|
||||||
},
|
},
|
||||||
"time": "2022-09-08T10:32:37+00:00"
|
"time": "2022-09-12T13:08:27+00:00"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"name": "utopia-php/domains",
|
"name": "utopia-php/domains",
|
||||||
|
|
|
@ -5,8 +5,10 @@ namespace Appwrite\Migration\Version;
|
||||||
use Appwrite\Migration\Migration;
|
use Appwrite\Migration\Migration;
|
||||||
use Appwrite\OpenSSL\OpenSSL;
|
use Appwrite\OpenSSL\OpenSSL;
|
||||||
use Exception;
|
use Exception;
|
||||||
|
use PDO;
|
||||||
use Utopia\App;
|
use Utopia\App;
|
||||||
use Utopia\CLI\Console;
|
use Utopia\CLI\Console;
|
||||||
|
use Utopia\Config\Config;
|
||||||
use Utopia\Database\Database;
|
use Utopia\Database\Database;
|
||||||
use Utopia\Database\Document;
|
use Utopia\Database\Document;
|
||||||
use Utopia\Database\ID;
|
use Utopia\Database\ID;
|
||||||
|
@ -20,11 +22,27 @@ class V15 extends Migration
|
||||||
*/
|
*/
|
||||||
private $pdo;
|
private $pdo;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @var array<string>
|
||||||
|
*/
|
||||||
|
protected array $providers;
|
||||||
|
|
||||||
public function execute(): void
|
public function execute(): void
|
||||||
{
|
{
|
||||||
global $register;
|
global $register;
|
||||||
$this->pdo = $register->get('db');
|
$this->pdo = $register->get('db');
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Populate providers.
|
||||||
|
*/
|
||||||
|
$this->providers = \array_merge(
|
||||||
|
['email', 'anonymous'],
|
||||||
|
\array_map(
|
||||||
|
fn ($value) => "oauth-" . $value,
|
||||||
|
\array_keys(Config::getParam('providers', []))
|
||||||
|
)
|
||||||
|
);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Disable SubQueries for Performance.
|
* Disable SubQueries for Performance.
|
||||||
*/
|
*/
|
||||||
|
@ -37,6 +55,12 @@ class V15 extends Migration
|
||||||
}
|
}
|
||||||
|
|
||||||
Console::log('Migrating Project: ' . $this->project->getAttribute('name') . ' (' . $this->project->getId() . ')');
|
Console::log('Migrating Project: ' . $this->project->getAttribute('name') . ' (' . $this->project->getId() . ')');
|
||||||
|
Console::info('Migrating Stats');
|
||||||
|
$this->migrateStatsMetric('requests', 'project.$all.network.requests');
|
||||||
|
$this->migrateStatsMetric('network', 'project.$all.network.bandwidth');
|
||||||
|
$this->migrateStatsMetric('executions', 'executions.$all.compute.total');
|
||||||
|
$this->migrateStatsMetric('storage.total', 'project.$all.storage.size');
|
||||||
|
|
||||||
Console::info('Migrating Collections');
|
Console::info('Migrating Collections');
|
||||||
$this->migrateCollections();
|
$this->migrateCollections();
|
||||||
Console::info('Migrating Databases');
|
Console::info('Migrating Databases');
|
||||||
|
@ -62,6 +86,21 @@ class V15 extends Migration
|
||||||
*/
|
*/
|
||||||
protected function migrateBuckets(): void
|
protected function migrateBuckets(): void
|
||||||
{
|
{
|
||||||
|
/**
|
||||||
|
* Migrating stats for all Buckets.
|
||||||
|
*/
|
||||||
|
$this->migrateStatsMetric('storage.files.total', 'files.$all.storage.size',);
|
||||||
|
$this->migrateStatsMetric('storage.files.count', 'files.$all.count.total');
|
||||||
|
$this->migrateStatsMetric('storage.buckets.count', 'buckets.$all.count.total');
|
||||||
|
$this->migrateStatsMetric('storage.buckets.create', 'buckets.$all.requests.create');
|
||||||
|
$this->migrateStatsMetric('storage.buckets.read', 'buckets.$all.requests.read');
|
||||||
|
$this->migrateStatsMetric('storage.buckets.update', 'buckets.$all.requests.update');
|
||||||
|
$this->migrateStatsMetric('storage.buckets.delete', 'buckets.$all.requests.delete');
|
||||||
|
$this->migrateStatsMetric('storage.files.create', 'files.$all.requests.create');
|
||||||
|
$this->migrateStatsMetric('storage.files.read', 'files.$all.requests.read');
|
||||||
|
$this->migrateStatsMetric('storage.files.update', 'files.$all.requests.update');
|
||||||
|
$this->migrateStatsMetric('storage.files.delete', 'files.$all.requests.delete');
|
||||||
|
|
||||||
foreach ($this->documentsIterator('buckets') as $bucket) {
|
foreach ($this->documentsIterator('buckets') as $bucket) {
|
||||||
$bucketTable = "bucket_{$bucket->getInternalId()}";
|
$bucketTable = "bucket_{$bucket->getInternalId()}";
|
||||||
|
|
||||||
|
@ -84,6 +123,17 @@ class V15 extends Migration
|
||||||
|
|
||||||
$this->projectDB->updateDocument('buckets', $bucket->getId(), $bucket);
|
$this->projectDB->updateDocument('buckets', $bucket->getId(), $bucket);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Migrating stats for every Bucket.
|
||||||
|
*/
|
||||||
|
$bucketId = $bucket->getId();
|
||||||
|
$this->migrateStatsMetric("storage.buckets.$bucketId.files.count", "files.$bucketId.count.total");
|
||||||
|
$this->migrateStatsMetric("storage.buckets.$bucketId.files.total", "files.$bucketId.storage.size");
|
||||||
|
$this->migrateStatsMetric("storage.buckets.$bucketId.files.create", "files.$bucketId.requests.create");
|
||||||
|
$this->migrateStatsMetric("storage.buckets.$bucketId.files.read", "files.$bucketId.requests.read");
|
||||||
|
$this->migrateStatsMetric("storage.buckets.$bucketId.files.update", "files.$bucketId.requests.update");
|
||||||
|
$this->migrateStatsMetric("storage.buckets.$bucketId.files.delete", "files.$bucketId.requests.delete");
|
||||||
|
|
||||||
Console::info("Migrating Files of {$bucket->getId()} ({$bucket->getAttribute('name')})");
|
Console::info("Migrating Files of {$bucket->getId()} ({$bucket->getAttribute('name')})");
|
||||||
foreach ($this->documentsIterator($bucketTable) as $file) {
|
foreach ($this->documentsIterator($bucketTable) as $file) {
|
||||||
$this->populatePermissionsAttribute(
|
$this->populatePermissionsAttribute(
|
||||||
|
@ -112,6 +162,28 @@ class V15 extends Migration
|
||||||
*/
|
*/
|
||||||
protected function migrateDatabases(): void
|
protected function migrateDatabases(): void
|
||||||
{
|
{
|
||||||
|
/**
|
||||||
|
* Migrating stats for all Databases.
|
||||||
|
*/
|
||||||
|
$this->migrateStatsMetric('databases.count', 'databases.$all.count.total');
|
||||||
|
$this->migrateStatsMetric('databases.documents.count', 'documents.$all.count.total');
|
||||||
|
$this->migrateStatsMetric('databases.collections.count', 'collections.$all.count.total');
|
||||||
|
$this->migrateStatsMetric('databases.create', 'databases.$all.requests.create');
|
||||||
|
$this->migrateStatsMetric('databases.read', 'databases.$all.requests.read');
|
||||||
|
$this->migrateStatsMetric('databases.update', 'databases.$all.requests.update');
|
||||||
|
$this->migrateStatsMetric('databases.delete', 'databases.$all.requests.delete');
|
||||||
|
$this->migrateStatsMetric('databases.collections.create', 'collections.$all.requests.create');
|
||||||
|
$this->migrateStatsMetric('databases.collections.read', 'collections.$all.requests.read');
|
||||||
|
$this->migrateStatsMetric('databases.collections.update', 'collections.$all.requests.update');
|
||||||
|
$this->migrateStatsMetric('databases.collections.delete', 'collections.$all.requests.delete');
|
||||||
|
$this->migrateStatsMetric('databases.documents.create', 'documents.$all.requests.create');
|
||||||
|
$this->migrateStatsMetric('databases.documents.read', 'documents.$all.requests.read');
|
||||||
|
$this->migrateStatsMetric('databases.documents.update', 'documents.$all.requests.update');
|
||||||
|
$this->migrateStatsMetric('databases.documents.delete', 'documents.$all.requests.delete');
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Migrate every Database.
|
||||||
|
*/
|
||||||
foreach ($this->documentsIterator('databases') as $database) {
|
foreach ($this->documentsIterator('databases') as $database) {
|
||||||
$databaseTable = "database_{$database->getInternalId()}";
|
$databaseTable = "database_{$database->getInternalId()}";
|
||||||
$this->createPermissionsColumn($databaseTable);
|
$this->createPermissionsColumn($databaseTable);
|
||||||
|
@ -131,6 +203,24 @@ class V15 extends Migration
|
||||||
Console::warning("'documentSecurity' from {$databaseTable}: {$th->getMessage()}");
|
Console::warning("'documentSecurity' from {$databaseTable}: {$th->getMessage()}");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Migrating stats for single Databases.
|
||||||
|
*/
|
||||||
|
$databaseId = $database->getId();
|
||||||
|
$this->migrateStatsMetric("databases.$databaseId.collections.count", "collections.$databaseId.count.total");
|
||||||
|
$this->migrateStatsMetric("databases.$databaseId.collections.create", "collections.$databaseId.requests.create");
|
||||||
|
$this->migrateStatsMetric("databases.$databaseId.collections.read", "collections.$databaseId.requests.read");
|
||||||
|
$this->migrateStatsMetric("databases.$databaseId.collections.update", "collections.$databaseId.requests.update");
|
||||||
|
$this->migrateStatsMetric("databases.$databaseId.collections.delete", "collections.$databaseId.requests.delete");
|
||||||
|
$this->migrateStatsMetric("databases.$databaseId.documents.count", "documents.$databaseId.count.total");
|
||||||
|
$this->migrateStatsMetric("databases.$databaseId.documents.create", "documents.$databaseId.requests.create");
|
||||||
|
$this->migrateStatsMetric("databases.$databaseId.documents.read", "documents.$databaseId.requests.read");
|
||||||
|
$this->migrateStatsMetric("databases.$databaseId.documents.update", "documents.$databaseId.requests.update");
|
||||||
|
$this->migrateStatsMetric("databases.$databaseId.documents.delete", "documents.$databaseId.requests.delete");
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Migrate every Collection.
|
||||||
|
*/
|
||||||
Console::info("Migrating Collections of {$database->getId()} ({$database->getAttribute('name')})");
|
Console::info("Migrating Collections of {$database->getId()} ({$database->getAttribute('name')})");
|
||||||
foreach ($this->documentsIterator($databaseTable) as $collection) {
|
foreach ($this->documentsIterator($databaseTable) as $collection) {
|
||||||
$collectionTable = "{$databaseTable}_collection_{$collection->getInternalId()}";
|
$collectionTable = "{$databaseTable}_collection_{$collection->getInternalId()}";
|
||||||
|
@ -150,6 +240,16 @@ class V15 extends Migration
|
||||||
|
|
||||||
$this->projectDB->updateDocument($databaseTable, $collection->getId(), $collection);
|
$this->projectDB->updateDocument($databaseTable, $collection->getId(), $collection);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Migrating stats for single Collections.
|
||||||
|
*/
|
||||||
|
$collectionId = $collection->getId();
|
||||||
|
$this->migrateStatsMetric("databases.{$databaseId}.collections.{$collectionId}.documents.count", "documents.{$databaseId}/{$collectionId}.count.total");
|
||||||
|
$this->migrateStatsMetric("databases.{$databaseId}.collections.{$collectionId}.documents.create", "documents.{$databaseId}/{$collectionId}.requests.create");
|
||||||
|
$this->migrateStatsMetric("databases.{$databaseId}.collections.{$collectionId}.documents.read", "documents.{$databaseId}/{$collectionId}.requests.read");
|
||||||
|
$this->migrateStatsMetric("databases.{$databaseId}.collections.{$collectionId}.documents.update", "documents.{$databaseId}/{$collectionId}.requests.update");
|
||||||
|
$this->migrateStatsMetric("databases.{$databaseId}.collections.{$collectionId}.documents.delete", "documents.{$databaseId}/{$collectionId}.requests.delete");
|
||||||
|
|
||||||
Console::info("Migrating Documents of {$collection->getId()} ({$collection->getAttribute('name')})");
|
Console::info("Migrating Documents of {$collection->getId()} ({$collection->getAttribute('name')})");
|
||||||
$requiredAttributes = array_reduce($collection->getAttribute('attributes', []), function (array $carry, Document $item) {
|
$requiredAttributes = array_reduce($collection->getAttribute('attributes', []), function (array $carry, Document $item) {
|
||||||
if ($item->getAttribute('required', false)) {
|
if ($item->getAttribute('required', false)) {
|
||||||
|
@ -630,7 +730,7 @@ class V15 extends Migration
|
||||||
*/
|
*/
|
||||||
Console::log("Migrating Collection \"{$id}\" Variables");
|
Console::log("Migrating Collection \"{$id}\" Variables");
|
||||||
|
|
||||||
foreach ($this->documentsIterator('functions') as $function) {
|
foreach ($this->documentsIterator($id) as $function) {
|
||||||
foreach ($function->getAttribute('vars', []) as $key => $value) {
|
foreach ($function->getAttribute('vars', []) as $key => $value) {
|
||||||
if ($value instanceof Document) {
|
if ($value instanceof Document) {
|
||||||
continue;
|
continue;
|
||||||
|
@ -1064,6 +1164,18 @@ class V15 extends Migration
|
||||||
Console::warning("'_key_phoneVerification' from {$id}: {$th->getMessage()}");
|
Console::warning("'_key_phoneVerification' from {$id}: {$th->getMessage()}");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
$this->migrateStatsMetric('users.count', 'users.$all.requests.count');
|
||||||
|
$this->migrateStatsMetric('users.create', 'users.$all.requests.create');
|
||||||
|
$this->migrateStatsMetric('users.read', 'users.$all.requests.read');
|
||||||
|
$this->migrateStatsMetric('users.update', 'users.$all.requests.update');
|
||||||
|
$this->migrateStatsMetric('users.delete', 'users.$all.requests.delete');
|
||||||
|
$this->migrateStatsMetric('users.sessions.create', 'sessions.$all.requests.create');
|
||||||
|
$this->migrateStatsMetric('users.sessions.delete', 'sessions.$all.requests.delete');
|
||||||
|
|
||||||
|
foreach ($this->providers as $provider) {
|
||||||
|
$this->migrateStatsMetric("users.sessions.{$provider}.create", "sessions.$provider.requests.create");
|
||||||
|
}
|
||||||
|
|
||||||
break;
|
break;
|
||||||
|
|
||||||
case 'webhooks':
|
case 'webhooks':
|
||||||
|
@ -1202,6 +1314,14 @@ class V15 extends Migration
|
||||||
$document->getAttribute('execute', [])
|
$document->getAttribute('execute', [])
|
||||||
));
|
));
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Migrate functions stats.
|
||||||
|
*/
|
||||||
|
$functionId = $document->getId();
|
||||||
|
$this->migrateStatsMetric("functions.$functionId.executions", "executions.$functionId.compute.total");
|
||||||
|
$this->migrateStatsMetric("functions.$functionId.failures", "executions.$functionId.compute.failure");
|
||||||
|
$this->migrateStatsMetric("functions.$functionId.compute", "executions.$functionId.compute.time");
|
||||||
|
|
||||||
break;
|
break;
|
||||||
|
|
||||||
case 'indexes':
|
case 'indexes':
|
||||||
|
@ -1307,6 +1427,18 @@ class V15 extends Migration
|
||||||
return $document;
|
return $document;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
protected function migrateStatsMetric(string $from, string $to): void
|
||||||
|
{
|
||||||
|
try {
|
||||||
|
$from = $this->pdo->quote($from);
|
||||||
|
$to = $this->pdo->quote($to);
|
||||||
|
|
||||||
|
$this->pdo->prepare("UPDATE `{$this->projectDB->getDefaultDatabase()}`.`_{$this->project->getInternalId()}_stats` SET metric = {$to} WHERE metric = {$from}")->execute();
|
||||||
|
} catch (\Throwable $th) {
|
||||||
|
Console::warning("Migrating steps from {$this->projectDB->getDefaultDatabase()}`.`_{$this->project->getInternalId()}_stats:" . $th->getMessage());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Filter from the 'encrypt' filter.
|
* Filter from the 'encrypt' filter.
|
||||||
*
|
*
|
||||||
|
|
Loading…
Reference in a new issue