diff --git a/src/Appwrite/Migration/Migration.php b/src/Appwrite/Migration/Migration.php index 9ea3443091..9c6fbe6b39 100644 --- a/src/Appwrite/Migration/Migration.php +++ b/src/Appwrite/Migration/Migration.php @@ -77,7 +77,11 @@ abstract class Migration Authorization::disable(); Authorization::setDefaultStatus(false); - $this->collections = array_merge([ + $this->collections = Config::getParam('collections', []); + + $projectCollections = $this->collections['projects']; + + $this->collections['projects'] = array_merge([ '_metadata' => [ '$id' => ID::custom('_metadata'), '$collection' => Database::METADATA @@ -90,7 +94,7 @@ abstract class Migration '$id' => ID::custom('abuse'), '$collection' => Database::METADATA ] - ], Config::getParam('collections', [])); + ], $projectCollections); } /** @@ -131,7 +135,14 @@ abstract class Migration */ public function forEachDocument(callable $callback): void { - foreach ($this->collections as $collection) { + $internalProjectId = $this->project->getInternalId(); + + $collections = match ($internalProjectId) { + 'console' => $this->collections['console'], + default => $this->collections['projects'], + }; + + foreach ($collections as $collection) { if ($collection['$collection'] !== Database::METADATA) { continue; } @@ -237,10 +248,15 @@ abstract class Migration { $name ??= $id; + $collectionType = match ($this->project->getInternalId()) { + 'console' => 'console', + default => 'projects', + }; + if (!$this->projectDB->exists(App::getEnv('_APP_DB_SCHEMA', 'appwrite'), $name)) { $attributes = []; $indexes = []; - $collection = $this->collections[$id]; + $collection = $this->collections[$collectionType][$id]; foreach ($collection['attributes'] as $attribute) { $attributes[] = new Document([ @@ -286,9 +302,16 @@ abstract class Migration public function createAttributeFromCollection(Database $database, string $collectionId, string $attributeId, string $from = null): void { $from ??= $collectionId; - $collection = Config::getParam('collections', [])[$from] ?? null; + $collectionType = match ($this->project->getInternalId()) { + 'console' => 'console', + default => 'projects', + }; + if ($from === 'files') { + $collectionType = 'buckets'; + } + $collection = $this->collections[$collectionType][$from] ?? null; if (is_null($collection)) { - throw new Exception("Collection {$collectionId} not found"); + throw new Exception("Collection {$from} not found"); } $attributes = $collection['attributes']; @@ -332,7 +355,11 @@ abstract class Migration public function createIndexFromCollection(Database $database, string $collectionId, string $indexId, string $from = null): void { $from ??= $collectionId; - $collection = Config::getParam('collections', [])[$collectionId] ?? null; + $collectionType = match ($this->project->getInternalId()) { + 'console' => 'console', + default => 'projects', + }; + $collection = $this->collections[$collectionType][$from] ?? null; if (is_null($collection)) { throw new Exception("Collection {$collectionId} not found"); diff --git a/src/Appwrite/Migration/Version/V19.php b/src/Appwrite/Migration/Version/V19.php index 44d0b308d5..74637c42ad 100644 --- a/src/Appwrite/Migration/Version/V19.php +++ b/src/Appwrite/Migration/Version/V19.php @@ -3,8 +3,11 @@ namespace Appwrite\Migration\Version; use Appwrite\Migration\Migration; +use Utopia\App; use Utopia\CLI\Console; +use Utopia\Config\Config; use Utopia\Database\Database; +use Utopia\Database\DateTime; use Utopia\Database\Document; class V19 extends Migration @@ -90,7 +93,7 @@ class V19 extends Migration Console::log("Migrating Collection \"{$id}\""); - $this->projectDB->setNamespace("_{$this->project->getInternalId()}"); + $this->projectDB->setNamespace("_$internalProjectId"); switch ($id) { case '_metadata': @@ -294,28 +297,20 @@ class V19 extends Migration // TODO: delete domains? break; - case 'schedules': - try { - $this->createAttributeFromCollection($this->projectDB, $id, 'resourceInternalId'); - $this->projectDB->deleteCachedCollection($id); - } catch (\Throwable $th) { - Console::warning("'resourceInternalId' from {$id}: {$th->getMessage()}"); - } - break; case 'stats': // TODO: should we do this now? try { $this->projectDB->updateAttribute($id, 'value', signed: true); $this->projectDB->deleteCachedCollection($id); } catch (\Throwable $th) { - Console::warning("'teamInternalId' from {$id}: {$th->getMessage()}"); + Console::warning("'value' from {$id}: {$th->getMessage()}"); } try { $this->projectDB->deleteAttribute($id, 'type'); $this->projectDB->deleteCachedCollection($id); } catch (\Throwable $th) { - Console::warning("'teamInternalId' from {$id}: {$th->getMessage()}"); + Console::warning("'type' from {$id}: {$th->getMessage()}"); } try { @@ -329,15 +324,7 @@ class V19 extends Migration $this->createIndexFromCollection($this->projectDB, $id, '_key_metric_period_time'); $this->projectDB->deleteCachedCollection($id); } catch (\Throwable $th) { - Console::warning("'teamInternalId' from {$id}: {$th->getMessage()}"); - } - break; - case 'teams': - try { - $this->projectDB->updateAttribute($id, 'teamInternalId', required: true); - $this->projectDB->deleteCachedCollection($id); - } catch (\Throwable $th) { - Console::warning("'teamInternalId' from {$id}: {$th->getMessage()}"); + Console::warning("'_key_metric_period_time' from {$id}: {$th->getMessage()}"); } break; case 'users': @@ -441,8 +428,6 @@ class V19 extends Migration { switch ($document->getCollection()) { case '_metadata': - // TODO: function schedules - // TODO: migrate statsLogger? break; case 'attributes': @@ -488,6 +473,7 @@ class V19 extends Migration $deploymentId = $document->getAttribute('deployment'); $deployment = $this->projectDB->getDocument('deployments', $deploymentId); $document->setAttribute('deploymentInternalId', $deployment->getInternalId()); + $document->setAttribute('entrypoint', $deployment->getAttribute('entrypoint')); $schedule = $this->consoleDB->createDocument('schedules', new Document([ 'region' => App::getEnv('_APP_REGION', 'default'), // Todo replace with projects region @@ -509,6 +495,10 @@ class V19 extends Migration */ $document->setAttribute('version', '1.4.0'); + $databases = Config::getParam('pools-database', []); + $database = $databases[0]; + + $document->setAttribute('database', $database); $document->setAttribute('smtp', []); $document->setAttribute('templates', []); @@ -567,7 +557,7 @@ class V19 extends Migration $this->alterUidType($id); try { - $this->createAttributeFromCollection($this->projectDB, $id, 'bucketInternalId'); + $this->createAttributeFromCollection($this->projectDB, $id, 'bucketInternalId', 'files'); $this->projectDB->deleteCachedCollection($id); } catch (\Throwable $th) { Console::warning("'bucketInternalId' from {$id}: {$th->getMessage()}"); diff --git a/src/Appwrite/Platform/Tasks/Migrate.php b/src/Appwrite/Platform/Tasks/Migrate.php index 90b4234109..38e5c8aa46 100644 --- a/src/Appwrite/Platform/Tasks/Migrate.php +++ b/src/Appwrite/Platform/Tasks/Migrate.php @@ -7,10 +7,10 @@ use Utopia\CLI\Console; use Appwrite\Migration\Migration; use Utopia\App; use Utopia\Cache\Cache; -use Utopia\Cache\Adapter\Redis as RedisCache; +use Utopia\Database\Database; use Utopia\Database\Document; +use Utopia\Database\Query; use Utopia\Database\Validator\Authorization; -use Utopia\Registry\Registry; use Utopia\Validator\Text; class Migrate extends Action @@ -26,20 +26,22 @@ class Migrate extends Action ->desc('Migrate Appwrite to new version') /** @TODO APP_VERSION_STABLE needs to be defined */ ->param('version', APP_VERSION_STABLE, new Text(8), 'Version to migrate to.', true) - ->inject('register') - ->callback(fn ($version, $register) => $this->action($version, $register)); + ->inject('cache') + ->inject('dbForConsole') + ->inject('getProjectDB') + ->callback(fn ($version, $cache, $dbForConsole, $getProjectDB) => $this->action($version, $cache, $dbForConsole, $getProjectDB)); } - private function clearProjectsCache(Redis $redis, Document $project) + private function clearProjectsCache(Cache $cache, Document $project) { try { - $redis->del($redis->keys("cache-_{$project->getInternalId()}:*")); + $cache->purge("cache-_{$project->getInternalId()}:*"); } catch (\Throwable $th) { Console::error('Failed to clear project ("' . $project->getId() . '") cache with error: ' . $th->getMessage()); } } - public function action(string $version, Registry $register) + public function action(string $version, Cache $cache, Database $dbForConsole, callable $getProjectDB) { Authorization::disable(); if (!array_key_exists($version, Migration::$versions)) { @@ -52,14 +54,6 @@ class Migrate extends Action Console::success('Starting Data Migration to version ' . $version); - $dbPool = $register->get('dbPool', true); - $redis = $register->get('cache', true); - - $cache = new Cache(new RedisCache($redis)); - - $dbForConsole = $dbPool->getDB('console', $cache); - $dbForConsole->setNamespace('_project_console'); - $console = $app->getResource('console'); $limit = 30; @@ -79,6 +73,7 @@ class Migrate extends Action } $class = 'Appwrite\\Migration\\Version\\' . Migration::$versions[$version]; + /** @var Migration $migration */ $migration = new $class(); while (!empty($projects)) { @@ -90,11 +85,11 @@ class Migrate extends Action continue; } - $this->clearProjectsCache($redis, $project); + $this->clearProjectsCache($cache, $project); try { // TODO: Iterate through all project DBs - $projectDB = $dbPool->getDB($project->getId(), $cache); + $projectDB = $getProjectDB($project); $migration ->setProject($project, $projectDB, $dbForConsole) ->execute(); @@ -103,11 +98,11 @@ class Migrate extends Action throw $th; } - $this->clearProjectsCache($redis, $project); + $this->clearProjectsCache($cache, $project); } $sum = \count($projects); - $projects = $dbForConsole->find('projects', limit: $limit, offset: $offset); + $projects = $dbForConsole->find('projects', [Query::limit($limit), Query::offset($offset)]); $offset = $offset + $limit; $count = $count + $sum; @@ -115,7 +110,6 @@ class Migrate extends Action Console::log('Migrated ' . $count . '/' . $totalProjects . ' projects...'); } - Swoole\Event::wait(); // Wait for Coroutines to finish Console::success('Data Migration Completed'); } }