feat: migration

app/tasks/migrate.php

@@ -46,7 +46,13 @@ $cli
         $offset = 0;
         $projects = [$console];
         $count = 0;
-        $totalProjects = $consoleDB->count('projects') + 1;
+
+        try {
+            $totalProjects = $consoleDB->count('projects') + 1;
+        } catch (\Throwable $th) {
+            $consoleDB->setNamespace('_console');
+            $totalProjects = $consoleDB->count('projects') + 1;
+        }
 
         $class = 'Appwrite\\Migration\\Version\\' . Migration::$versions[$version];
         $migration = new $class();

src/Appwrite/Migration/Migration.php

@@ -38,6 +38,26 @@ abstract class Migration
         '0.13.0' => 'V12',
     ];
 
+    /**
+     * @var array
+     */
+    protected array $collections;
+
+    public function __construct()
+    {
+        $this->collections = array_merge([
+            '_metadata' => [
+                '$id' => '_metadata'
+            ],
+            'audit' => [
+                '$id' => 'audit'
+            ],
+            'abuse' => [
+                '$id' => 'abuse'
+            ]
+        ], Config::getParam('collections', []));
+    }
+
     /**
      * Set project for migration.
      *
@@ -51,7 +71,7 @@ abstract class Migration
     {
         $this->project = $project;
         $this->projectDB = $projectDB;
-        $this->projectDB->setNamespace('_project_' . $this->project->getId());
+        $this->projectDB->setNamespace('_' . $this->project->getId());
 
         $this->consoleDB = $consoleDB;
 
@@ -67,10 +87,7 @@ abstract class Migration
     {
         Runtime::enableCoroutine(SWOOLE_HOOK_ALL);
 
-        /** @var array $collections */
-        $collections = Config::getParam('collections', []);
-
-        foreach ($collections as $collection) {
+        foreach ($this->collections as $collection) {
             $sum = 0;
             $nextDocument = null;
             $collectionCount = $this->projectDB->count($collection['$id']);
@@ -114,6 +131,7 @@ abstract class Migration
                             try {
                                 $new = $this->projectDB->updateDocument($document->getCollection(), $document->getId(), $document);
                             } catch (\Throwable $th) {
+                                var_dump($th->getTraceAsString());
                                 Console::error('Failed to update document: ' . $th->getMessage());
                                 return;
 

src/Appwrite/Migration/Version/V12.php

@@ -4,17 +4,178 @@ namespace Appwrite\Migration\Version;
 
 use Appwrite\Migration\Migration;
 use Utopia\CLI\Console;
+use Utopia\Database\Database;
 use Utopia\Database\Document;
 
 class V12 extends Migration
 {
+    /**
+     * @var \PDO $pdo
+     */
+    private $pdo;
+
     public function execute(): void
     {
+        global $register;
         Console::log('Migrating project: ' . $this->project->getAttribute('name') . ' (' . $this->project->getId() . ')');
 
+        $this->pdo = $register->get('db');
+
+        Console::info('Migrating Project Schemas');
+        $this->migrateProjectSchema($this->project->getId());
+
+        /**
+         * Switch to migrated Console Project
+         */
+        if ($this->project->getId() === 'console') {
+            $this->consoleDB->setNamespace('_console');
+            $this->projectDB->setNamespace('_console');
+        }
+
+        Console::info('Migrating Permissions');
+        $this->fixPermissions();
+        Console::info('Migrating Collections');
+        $this->fixCollections();
+        Console::info('Migrating Documents');
         $this->forEachDocument([$this, 'fixDocument']);
     }
 
+    /**
+     * Migrate Project Tables.
+     *
+     * @param string $projectId
+     * @return void
+     * @throws \Exception
+     * @throws \PDOException
+     */
+    private function migrateProjectSchema(string $projectId): void
+    {
+        /**
+         * Remove empty generated Console Project.
+         */
+        if ($this->projectDB->getNamespace() === '_project_console') {
+            $all = [];
+            foreach ($this->collections as $collection) {
+                $all[] = "_{$projectId}_{$collection['$id']}";
+                $all[] = "_{$projectId}_{$collection['$id']}_perms";
+            }
+            $this->pdo->prepare('DROP TABLE IF EXISTS ' . implode(', ', $all) . ';')->execute();
+        } elseif ($this->projectDB->getNamespace() === '_console') {
+            return;
+        }
+
+        /**
+         * Rename Database Tables.
+         */
+        foreach ($this->collections as $collection) {
+            $id = $collection['$id'];
+
+            $this->pdo->prepare("ALTER TABLE IF EXISTS _project_{$projectId}_{$id} RENAME TO _{$projectId}_{$id}")->execute();
+            $this->pdo->prepare("CREATE TABLE IF NOT EXISTS _{$projectId}_{$id}_perms (
+                `_id` int(11) unsigned NOT NULL AUTO_INCREMENT,
+                `_type` VARCHAR(12) NOT NULL,
+                `_permission` VARCHAR(255) NOT NULL,
+                `_document` VARCHAR(255) NOT NULL,
+                PRIMARY KEY (`_id`),
+                UNIQUE INDEX `_index1` (`_type`,`_document`,`_permission`),
+                INDEX `_index2` (`_permission`)
+            ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;")->execute();
+        }
+    }
+
+    /**
+     * Migrate all Collection Structure.
+     *
+     * @return void
+     */
+    protected function fixCollections(): void
+    {
+        foreach ($this->collections as $collection) {
+            $id = $collection['$id'];
+            Console::log("- {$id}");
+            switch ($id) {
+                case 'sessions':
+                    try {
+                        $this->projectDB->renameAttribute($id, 'providerToken', 'providerAccessToken');
+                    } catch (\Throwable $th) {
+                        Console::warning("'providerAccessToken' from {$id}: {$th->getMessage()}");
+                    }
+                    try {
+                        $this->projectDB->createAttribute(collection: $id, id: 'providerRefreshToken', type: Database::VAR_STRING, size: 16384, signed: true, required: true, filters: ['encrypt']);
+                    } catch (\Throwable $th) {
+                        Console::warning("'providerRefreshToken' from {$id}: {$th->getMessage()}");
+                    }
+                    try {
+                        $this->projectDB->createAttribute(collection: $id, id: 'providerAccessTokenExpiry', type: Database::VAR_INTEGER, size: 0, required: true);
+                    } catch (\Throwable $th) {
+                        Console::warning("'providerAccessTokenExpiry' from {$id}: {$th->getMessage()}");
+                    }
+                    break;
+            }
+            usleep(100000);
+        }
+    }
+
+    /**
+     * Migrate all Permission to new System with dedicated Table.
+     * @return void
+     * @throws \Exception
+     */
+    protected function fixPermissions()
+    {
+        foreach ($this->collections as $collection) {
+            $id = $collection['$id'];
+            Console::log("- {$collection['$id']}");
+            $nextDocument = null;
+
+            do {
+                $documents = $this->projectDB->find($id, limit: $this->limit, cursor: $nextDocument);
+                $count = count($documents);
+
+                \Co\run(function (array $documents) {
+                    foreach ($documents as $document) {
+                        go(function (Document $document) {
+                            $sql = "SELECT _read, _write FROM `{$this->projectDB->getDefaultDatabase()}`.`{$this->projectDB->getNamespace()}_{$document->getCollection()}` WHERE _uid = {$this->pdo->quote($document->getid())}";
+                            $stmt = $this->pdo->prepare($sql);
+                            $stmt->execute();
+
+                            $permissions = $stmt->fetch();
+
+                            $read  = json_decode($permissions['_read'] ?? null) ?? [];
+                            $write = json_decode($permissions['_write'] ?? null) ?? [];
+
+                            $permissions = [];
+                            foreach ($read as $permission) {
+                                $permissions[] = "('read', '{$permission}', '{$document->getId()}')";
+                            }
+
+                            foreach ($write as $permission) {
+                                $permissions[] = "('write', '{$permission}', '{$document->getId()}')";
+                            }
+
+                            if (!empty($permissions)) {
+                                $queryPermissions = "INSERT IGNORE INTO `{$this->projectDB->getDefaultDatabase()}`.`{$this->projectDB->getNamespace()}_{$document->getCollection()}_perms` (_type, _permission, _document) VALUES " . implode(', ', $permissions);
+                                $stmtPermissions = $this->pdo->prepare($queryPermissions);
+                                $stmtPermissions->execute();
+                            }
+                        }, $document);
+                    }
+                }, $documents);
+
+                if ($count !== $this->limit) {
+                    $nextDocument = null;
+                } else {
+                    $nextDocument = end($documents);
+                }
+            } while (!is_null($nextDocument));
+        }
+
+        /**
+         * Timeout to give MariaDB some room to breath
+         */
+        usleep(100000);
+    }
+
     protected function fixDocument(Document $document)
     {
         switch ($document->getCollection()) {
@@ -92,6 +253,15 @@ class V12 extends Migration
                 }
 
                 break;
+
+            case 'sessions':
+                $document
+                    ->setAttribute('providerRefreshToken', '')
+                    ->setAttribute('providerAccessTokenExpiry', 0)
+                    ->setAttribute('providerAccessToken', $document->getAttribute('providerToken', ''))
+                    ->removeAttribute('providerToken');
+
+                break;
         }
 
         return $document;