CodeMirror/appwrite
1
0
Fork 0
mirror of synced 2024-10-02 10:16:27 +13:00
Code Issues Projects Releases Wiki Activity

Fix Migration task after it was converted to a Platform task

Browse source
This commit is contained in:
Steven Nguyen 2023-08-15 17:37:16 -07:00
parent 586ad1984c
commit 436600547c
No known key found for this signature in database
3 changed files with 61 additions and 50 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

41
src/Appwrite/Migration/Migration.php
View file

@ -77,7 +77,11 @@ abstract class Migration
Authorization::disable();
Authorization::setDefaultStatus(false);
$this->collections = array_merge([
$this->collections = Config::getParam('collections', []);
$projectCollections = $this->collections['projects'];
$this->collections['projects'] = array_merge([
'_metadata' => [
'$id' => ID::custom('_metadata'),
'$collection' => Database::METADATA
@ -90,7 +94,7 @@ abstract class Migration
'$id' => ID::custom('abuse'),
'$collection' => Database::METADATA
]
], Config::getParam('collections', []));
], $projectCollections);
}
/**
@ -131,7 +135,14 @@ abstract class Migration
*/
public function forEachDocument(callable $callback): void
{
foreach ($this->collections as $collection) {
$internalProjectId = $this->project->getInternalId();
$collections = match ($internalProjectId) {
'console' => $this->collections['console'],
default => $this->collections['projects'],
};
foreach ($collections as $collection) {
if ($collection['$collection'] !== Database::METADATA) {
continue;
}
@ -237,10 +248,15 @@ abstract class Migration
{
$name ??= $id;
$collectionType = match ($this->project->getInternalId()) {
'console' => 'console',
default => 'projects',
};
if (!$this->projectDB->exists(App::getEnv('_APP_DB_SCHEMA', 'appwrite'), $name)) {
$attributes = [];
$indexes = [];
$collection = $this->collections[$id];
$collection = $this->collections[$collectionType][$id];
foreach ($collection['attributes'] as $attribute) {
$attributes[] = new Document([
@ -286,9 +302,16 @@ abstract class Migration
public function createAttributeFromCollection(Database $database, string $collectionId, string $attributeId, string $from = null): void
{
$from ??= $collectionId;
$collection = Config::getParam('collections', [])[$from] ?? null;
$collectionType = match ($this->project->getInternalId()) {
'console' => 'console',
default => 'projects',
};
if ($from === 'files') {
$collectionType = 'buckets';
}
$collection = $this->collections[$collectionType][$from] ?? null;
if (is_null($collection)) {
throw new Exception("Collection {$collectionId} not found");
throw new Exception("Collection {$from} not found");
}
$attributes = $collection['attributes'];
@ -332,7 +355,11 @@ abstract class Migration
public function createIndexFromCollection(Database $database, string $collectionId, string $indexId, string $from = null): void
{
$from ??= $collectionId;
$collection = Config::getParam('collections', [])[$collectionId] ?? null;
$collectionType = match ($this->project->getInternalId()) {
'console' => 'console',
default => 'projects',
};
$collection = $this->collections[$collectionType][$from] ?? null;
if (is_null($collection)) {
throw new Exception("Collection {$collectionId} not found");

36
src/Appwrite/Migration/Version/V19.php
View file

@ -3,8 +3,11 @@
namespace Appwrite\Migration\Version;
use Appwrite\Migration\Migration;
use Utopia\App;
use Utopia\CLI\Console;
use Utopia\Config\Config;
use Utopia\Database\Database;
use Utopia\Database\DateTime;
use Utopia\Database\Document;
class V19 extends Migration
@ -90,7 +93,7 @@ class V19 extends Migration
Console::log("Migrating Collection \"{$id}\"");
$this->projectDB->setNamespace("_{$this->project->getInternalId()}");
$this->projectDB->setNamespace("_$internalProjectId");
switch ($id) {
case '_metadata':
@ -294,28 +297,20 @@ class V19 extends Migration
// TODO: delete domains?
break;
case 'schedules':
try {
$this->createAttributeFromCollection($this->projectDB, $id, 'resourceInternalId');
$this->projectDB->deleteCachedCollection($id);
} catch (\Throwable $th) {
Console::warning("'resourceInternalId' from {$id}: {$th->getMessage()}");
}
break;
case 'stats':
// TODO: should we do this now?
try {
$this->projectDB->updateAttribute($id, 'value', signed: true);
$this->projectDB->deleteCachedCollection($id);
} catch (\Throwable $th) {
Console::warning("'teamInternalId' from {$id}: {$th->getMessage()}");
Console::warning("'value' from {$id}: {$th->getMessage()}");
}
try {
$this->projectDB->deleteAttribute($id, 'type');
$this->projectDB->deleteCachedCollection($id);
} catch (\Throwable $th) {
Console::warning("'teamInternalId' from {$id}: {$th->getMessage()}");
Console::warning("'type' from {$id}: {$th->getMessage()}");
}
try {
@ -329,15 +324,7 @@ class V19 extends Migration
$this->createIndexFromCollection($this->projectDB, $id, '_key_metric_period_time');
$this->projectDB->deleteCachedCollection($id);
} catch (\Throwable $th) {
Console::warning("'teamInternalId' from {$id}: {$th->getMessage()}");
}
break;
case 'teams':
try {
$this->projectDB->updateAttribute($id, 'teamInternalId', required: true);
$this->projectDB->deleteCachedCollection($id);
} catch (\Throwable $th) {
Console::warning("'teamInternalId' from {$id}: {$th->getMessage()}");
Console::warning("'_key_metric_period_time' from {$id}: {$th->getMessage()}");
}
break;
case 'users':
@ -441,8 +428,6 @@ class V19 extends Migration
{
switch ($document->getCollection()) {
case '_metadata':
// TODO: function schedules
// TODO: migrate statsLogger?
break;
case 'attributes':
@ -488,6 +473,7 @@ class V19 extends Migration
$deploymentId = $document->getAttribute('deployment');
$deployment = $this->projectDB->getDocument('deployments', $deploymentId);
$document->setAttribute('deploymentInternalId', $deployment->getInternalId());
$document->setAttribute('entrypoint', $deployment->getAttribute('entrypoint'));
$schedule = $this->consoleDB->createDocument('schedules', new Document([
'region' => App::getEnv('_APP_REGION', 'default'), // Todo replace with projects region
@ -509,6 +495,10 @@ class V19 extends Migration
*/
$document->setAttribute('version', '1.4.0');
$databases = Config::getParam('pools-database', []);
$database = $databases[0];
$document->setAttribute('database', $database);
$document->setAttribute('smtp', []);
$document->setAttribute('templates', []);
@ -567,7 +557,7 @@ class V19 extends Migration
$this->alterUidType($id);
try {
$this->createAttributeFromCollection($this->projectDB, $id, 'bucketInternalId');
$this->createAttributeFromCollection($this->projectDB, $id, 'bucketInternalId', 'files');
$this->projectDB->deleteCachedCollection($id);
} catch (\Throwable $th) {
Console::warning("'bucketInternalId' from {$id}: {$th->getMessage()}");

34
src/Appwrite/Platform/Tasks/Migrate.php
View file

@ -7,10 +7,10 @@ use Utopia\CLI\Console;
use Appwrite\Migration\Migration;
use Utopia\App;
use Utopia\Cache\Cache;
use Utopia\Cache\Adapter\Redis as RedisCache;
use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\Database\Query;
use Utopia\Database\Validator\Authorization;
use Utopia\Registry\Registry;
use Utopia\Validator\Text;
class Migrate extends Action
@ -26,20 +26,22 @@ class Migrate extends Action
->desc('Migrate Appwrite to new version')
/** @TODO APP_VERSION_STABLE needs to be defined */
->param('version', APP_VERSION_STABLE, new Text(8), 'Version to migrate to.', true)
->inject('register')
->callback(fn ($version, $register) => $this->action($version, $register));
->inject('cache')
->inject('dbForConsole')
->inject('getProjectDB')
->callback(fn ($version, $cache, $dbForConsole, $getProjectDB) => $this->action($version, $cache, $dbForConsole, $getProjectDB));
}
private function clearProjectsCache(Redis $redis, Document $project)
private function clearProjectsCache(Cache $cache, Document $project)
{
try {
$redis->del($redis->keys("cache-_{$project->getInternalId()}:*"));
$cache->purge("cache-_{$project->getInternalId()}:*");
} catch (\Throwable $th) {
Console::error('Failed to clear project ("' . $project->getId() . '") cache with error: ' . $th->getMessage());
}
}
public function action(string $version, Registry $register)
public function action(string $version, Cache $cache, Database $dbForConsole, callable $getProjectDB)
{
Authorization::disable();
if (!array_key_exists($version, Migration::$versions)) {
@ -52,14 +54,6 @@ class Migrate extends Action
Console::success('Starting Data Migration to version ' . $version);
$dbPool = $register->get('dbPool', true);
$redis = $register->get('cache', true);
$cache = new Cache(new RedisCache($redis));
$dbForConsole = $dbPool->getDB('console', $cache);
$dbForConsole->setNamespace('_project_console');
$console = $app->getResource('console');
$limit = 30;
@ -79,6 +73,7 @@ class Migrate extends Action
}
$class = 'Appwrite\\Migration\\Version\\' . Migration::$versions[$version];
/** @var Migration $migration */
$migration = new $class();
while (!empty($projects)) {
@ -90,11 +85,11 @@ class Migrate extends Action
continue;
}
$this->clearProjectsCache($redis, $project);
$this->clearProjectsCache($cache, $project);
try {
// TODO: Iterate through all project DBs
$projectDB = $dbPool->getDB($project->getId(), $cache);
$projectDB = $getProjectDB($project);
$migration
->setProject($project, $projectDB, $dbForConsole)
->execute();
@ -103,11 +98,11 @@ class Migrate extends Action
throw $th;
}
$this->clearProjectsCache($redis, $project);
$this->clearProjectsCache($cache, $project);
}
$sum = \count($projects);
$projects = $dbForConsole->find('projects', limit: $limit, offset: $offset);
$projects = $dbForConsole->find('projects', [Query::limit($limit), Query::offset($offset)]);
$offset = $offset + $limit;
$count = $count + $sum;
@ -115,7 +110,6 @@ class Migrate extends Action
Console::log('Migrated ' . $count . '/' . $totalProjects . ' projects...');
}
Swoole\Event::wait(); // Wait for Coroutines to finish
Console::success('Data Migration Completed');
}
}