feat(migration): v06 - first draft
This commit is contained in:
parent
3da045f6f7
commit
c6f9f2fc1e
|
@ -5,187 +5,26 @@ global $cli, $register, $projectDB, $console;
|
|||
use Utopia\Config\Config;
|
||||
use Utopia\CLI\Console;
|
||||
use Appwrite\Database\Database;
|
||||
use Appwrite\Database\Document;
|
||||
use Appwrite\Database\Validator\Authorization;
|
||||
use Appwrite\Database\Adapter\MySQL as MySQLAdapter;
|
||||
use Appwrite\Database\Adapter\Redis as RedisAdapter;
|
||||
|
||||
$callbacks = [
|
||||
'0.4.0' => function() {
|
||||
Console::log('I got nothing to do.');
|
||||
},
|
||||
|
||||
'0.5.0' => function($project) use ($register, $projectDB) {
|
||||
$db = $register->get('db');
|
||||
|
||||
Console::log('Migrating project: '.$project->getAttribute('name').' ('.$project->getId().')');
|
||||
|
||||
// Update all documents $uid -> $id
|
||||
|
||||
$limit = 30;
|
||||
$sum = 30;
|
||||
$offset = 0;
|
||||
|
||||
while ($sum >= 30) {
|
||||
$all = $projectDB->getCollection([
|
||||
'limit' => $limit,
|
||||
'offset' => $offset,
|
||||
'orderType' => 'DESC',
|
||||
]);
|
||||
|
||||
$sum = \count($all);
|
||||
|
||||
Console::log('Migrating: '.$offset.' / '.$projectDB->getSum());
|
||||
|
||||
foreach($all as $document) {
|
||||
$document = fixDocument($document);
|
||||
|
||||
if(empty($document->getId())) {
|
||||
throw new Exception('Missing ID');
|
||||
}
|
||||
|
||||
try {
|
||||
$new = $projectDB->overwriteDocument($document->getArrayCopy());
|
||||
} catch (\Throwable $th) {
|
||||
var_dump($document);
|
||||
Console::error('Failed to update document: '.$th->getMessage());
|
||||
continue;
|
||||
}
|
||||
|
||||
if($new->getId() !== $document->getId()) {
|
||||
throw new Exception('Duplication Error');
|
||||
}
|
||||
}
|
||||
|
||||
$offset = $offset + $limit;
|
||||
}
|
||||
|
||||
$schema = $_SERVER['_APP_DB_SCHEMA'] ?? '';
|
||||
|
||||
try {
|
||||
$statement = $db->prepare("
|
||||
|
||||
CREATE TABLE IF NOT EXISTS `template.database.unique` (
|
||||
`id` int(10) unsigned NOT NULL AUTO_INCREMENT,
|
||||
`key` varchar(128) DEFAULT NULL,
|
||||
PRIMARY KEY (`id`),
|
||||
UNIQUE KEY `index1` (`key`)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS `{$schema}`.`app_{$project->getId()}.database.unique` LIKE `template.database.unique`;
|
||||
ALTER TABLE `{$schema}`.`app_{$project->getId()}.audit.audit` DROP COLUMN IF EXISTS `userType`;
|
||||
ALTER TABLE `{$schema}`.`app_{$project->getId()}.audit.audit` DROP INDEX IF EXISTS `index_1`;
|
||||
ALTER TABLE `{$schema}`.`app_{$project->getId()}.audit.audit` ADD INDEX IF NOT EXISTS `index_1` (`userId` ASC);
|
||||
");
|
||||
|
||||
$statement->closeCursor();
|
||||
|
||||
$statement->execute();
|
||||
}
|
||||
catch (\Exception $e) {
|
||||
Console::error('Failed to alter table for project: '.$project->getId().' with message: '.$e->getMessage().'/');
|
||||
}
|
||||
},
|
||||
];
|
||||
|
||||
function fixDocument(Document $document) {
|
||||
$providers = Config::getParam('providers');
|
||||
|
||||
if($document->getAttribute('$collection') === Database::SYSTEM_COLLECTION_PROJECTS){
|
||||
foreach($providers as $key => $provider) {
|
||||
if(!empty($document->getAttribute('usersOauth'.\ucfirst($key).'Appid'))) {
|
||||
$document
|
||||
->setAttribute('usersOauth2'.\ucfirst($key).'Appid', $document->getAttribute('usersOauth'.\ucfirst($key).'Appid', ''))
|
||||
->removeAttribute('usersOauth'.\ucfirst($key).'Appid')
|
||||
;
|
||||
}
|
||||
|
||||
if(!empty($document->getAttribute('usersOauth'.\ucfirst($key).'Secret'))) {
|
||||
$document
|
||||
->setAttribute('usersOauth2'.\ucfirst($key).'Secret', $document->getAttribute('usersOauth'.\ucfirst($key).'Secret', ''))
|
||||
->removeAttribute('usersOauth'.\ucfirst($key).'Secret')
|
||||
;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if($document->getAttribute('$collection') === Database::SYSTEM_COLLECTION_WEBHOOKS){
|
||||
$document->setAttribute('security', ($document->getAttribute('security')) ? true : false);
|
||||
}
|
||||
|
||||
if($document->getAttribute('$collection') === Database::SYSTEM_COLLECTION_TASKS){
|
||||
$document->setAttribute('security', ($document->getAttribute('security')) ? true : false);
|
||||
}
|
||||
|
||||
if($document->getAttribute('$collection') === Database::SYSTEM_COLLECTION_USERS) {
|
||||
foreach($providers as $key => $provider) {
|
||||
if(!empty($document->getAttribute('oauth'.\ucfirst($key)))) {
|
||||
$document
|
||||
->setAttribute('oauth2'.\ucfirst($key), $document->getAttribute('oauth'.\ucfirst($key), ''))
|
||||
->removeAttribute('oauth'.\ucfirst($key))
|
||||
;
|
||||
}
|
||||
|
||||
if(!empty($document->getAttribute('oauth'.\ucfirst($key).'AccessToken'))) {
|
||||
$document
|
||||
->setAttribute('oauth2'.\ucfirst($key).'AccessToken', $document->getAttribute('oauth'.\ucfirst($key).'AccessToken', ''))
|
||||
->removeAttribute('oauth'.\ucfirst($key).'AccessToken')
|
||||
;
|
||||
}
|
||||
}
|
||||
|
||||
if($document->getAttribute('confirm', null) !== null) {
|
||||
$document
|
||||
->setAttribute('emailVerification', $document->getAttribute('confirm', $document->getAttribute('emailVerification', false)))
|
||||
->removeAttribute('confirm')
|
||||
;
|
||||
}
|
||||
}
|
||||
|
||||
if($document->getAttribute('$collection') === Database::SYSTEM_COLLECTION_PLATFORMS) {
|
||||
if($document->getAttribute('url', null) !== null) {
|
||||
$document
|
||||
->setAttribute('hostname', \parse_url($document->getAttribute('url', $document->getAttribute('hostname', '')), PHP_URL_HOST))
|
||||
->removeAttribute('url')
|
||||
;
|
||||
}
|
||||
}
|
||||
|
||||
$document
|
||||
->setAttribute('$id', $document->getAttribute('$uid', $document->getAttribute('$id')))
|
||||
->removeAttribute('$uid')
|
||||
;
|
||||
|
||||
foreach($document as &$attr) { // Handle child documents
|
||||
if($attr instanceof Document) {
|
||||
$attr = fixDocument($attr);
|
||||
}
|
||||
|
||||
if(\is_array($attr)) {
|
||||
foreach($attr as &$child) {
|
||||
if($child instanceof Document) {
|
||||
$child = fixDocument($child);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return $document;
|
||||
}
|
||||
use Appwrite\Migration\Version;
|
||||
|
||||
$cli
|
||||
->task('migrate')
|
||||
->action(function () use ($register, $callbacks) {
|
||||
->action(function () use ($register) {
|
||||
Console::success('Starting Data Migration');
|
||||
|
||||
$consoleDB = new Database();
|
||||
$consoleDB->setAdapter(new RedisAdapter(new MySQLAdapter($register), $register));
|
||||
$consoleDB->setNamespace('app_console'); // Main DB
|
||||
$consoleDB->setMocks(Config::getParam('collections', []));
|
||||
|
||||
$consoleDB
|
||||
->setAdapter(new RedisAdapter(new MySQLAdapter($register), $register))
|
||||
->setNamespace('app_console') // Main DB
|
||||
->setMocks(Config::getParam('collections', []));
|
||||
|
||||
$projectDB = new Database();
|
||||
$projectDB->setAdapter(new RedisAdapter(new MySQLAdapter($register), $register));
|
||||
$projectDB->setMocks(Config::getParam('collections', []));
|
||||
$projectDB
|
||||
->setAdapter(new RedisAdapter(new MySQLAdapter($register), $register))
|
||||
->setMocks(Config::getParam('collections', []));
|
||||
|
||||
$console = $consoleDB->getDocument('console');
|
||||
|
||||
|
@ -197,16 +36,17 @@ $cli
|
|||
$projects = [$console];
|
||||
$count = 0;
|
||||
|
||||
while ($sum >= 30) {
|
||||
foreach($projects as $project) {
|
||||
|
||||
$projectDB->setNamespace('app_'.$project->getId());
|
||||
$migration = new Version\V06($register->get('db')); //TODO: remove hardcoded version and move to dynamic migration
|
||||
|
||||
while ($sum >= 30) {
|
||||
foreach ($projects as $project) {
|
||||
try {
|
||||
$callbacks['0.5.0']($project, $projectDB);
|
||||
$migration
|
||||
->setProject($project, $projectDB)
|
||||
->execute();
|
||||
} catch (\Throwable $th) {
|
||||
throw $th;
|
||||
Console::error('Failed to update project ("'.$project->getId().'") version with error: '.$th->getMessage());
|
||||
Console::error('Failed to update project ("' . $project->getId() . '") version with error: ' . $th->getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -214,7 +54,7 @@ $cli
|
|||
'limit' => $limit,
|
||||
'offset' => $offset,
|
||||
'filters' => [
|
||||
'$collection='.Database::SYSTEM_COLLECTION_PROJECTS,
|
||||
'$collection=' . Database::SYSTEM_COLLECTION_PROJECTS,
|
||||
],
|
||||
]);
|
||||
|
||||
|
@ -222,8 +62,8 @@ $cli
|
|||
$offset = $offset + $limit;
|
||||
$count = $count + $sum;
|
||||
|
||||
Console::log('Fetched '.$count.'/'.$consoleDB->getSum().' projects...');
|
||||
Console::log('Fetched ' . $count . '/' . $consoleDB->getSum() . ' projects...');
|
||||
}
|
||||
|
||||
Console::success('Data Migration Completed');
|
||||
});
|
||||
});
|
||||
|
|
41
src/Appwrite/Migration/Migration.php
Normal file
41
src/Appwrite/Migration/Migration.php
Normal file
|
@ -0,0 +1,41 @@
|
|||
<?php
|
||||
|
||||
namespace Appwrite\Migration;
|
||||
|
||||
use Appwrite\Database\Document;
|
||||
use Appwrite\Database\Database;
|
||||
|
||||
abstract class Migration
|
||||
{
|
||||
protected \PDO $db;
|
||||
|
||||
protected int $limit = 30;
|
||||
protected int $sum = 30;
|
||||
protected int $offset = 0;
|
||||
protected Document $project;
|
||||
protected Database $projectDB;
|
||||
|
||||
/**
|
||||
* Migration constructor.
|
||||
*/
|
||||
public function __construct(\PDO $db)
|
||||
{
|
||||
$this->db = $db;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set project for migration.
|
||||
*/
|
||||
public function setProject(Document $project, Database $projectDB)
|
||||
{
|
||||
$this->project = $project;
|
||||
$this->projectDB = $projectDB;
|
||||
$this->projectDB->setNamespace('app_'.$project->getId());
|
||||
return $this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Executes migration for set project.
|
||||
*/
|
||||
abstract public function execute(): void;
|
||||
}
|
14
src/Appwrite/Migration/Version/V04.php
Normal file
14
src/Appwrite/Migration/Version/V04.php
Normal file
|
@ -0,0 +1,14 @@
|
|||
<?php
|
||||
|
||||
namespace Appwrite\Migration\Version;
|
||||
|
||||
use Utopia\CLI\Console;
|
||||
use Appwrite\Migration\Migration;
|
||||
|
||||
class V04 extends Migration
|
||||
{
|
||||
public function execute(): void
|
||||
{
|
||||
Console::log('I got nothing to do.');
|
||||
}
|
||||
}
|
164
src/Appwrite/Migration/Version/V05.php
Normal file
164
src/Appwrite/Migration/Version/V05.php
Normal file
|
@ -0,0 +1,164 @@
|
|||
<?php
|
||||
|
||||
namespace Appwrite\Migration\Version;
|
||||
|
||||
use Appwrite\Migration\Migration;
|
||||
use Utopia\Config\Config;
|
||||
use Utopia\CLI\Console;
|
||||
use Utopia\Exception;
|
||||
use Appwrite\Database\Database;
|
||||
use Appwrite\Database\Document;
|
||||
|
||||
class V05 extends Migration
|
||||
{
|
||||
public function execute(): void
|
||||
{
|
||||
$db = $this->db;
|
||||
$project = $this->project;
|
||||
$projectDB = $this->projectDB;
|
||||
Console::log('Migrating project: ' . $project->getAttribute('name') . ' (' . $project->getId() . ')');
|
||||
|
||||
// Update all documents $uid -> $id
|
||||
|
||||
$limit = 30;
|
||||
$sum = 30;
|
||||
$offset = 0;
|
||||
|
||||
while ($sum >= 30) {
|
||||
$all = $projectDB->getCollection([
|
||||
'limit' => $limit,
|
||||
'offset' => $offset,
|
||||
'orderType' => 'DESC',
|
||||
]);
|
||||
|
||||
$sum = \count($all);
|
||||
|
||||
Console::log('Migrating: ' . $offset . ' / ' . $projectDB->getSum());
|
||||
|
||||
foreach ($all as $document) {
|
||||
$document = $this->fixDocument($document);
|
||||
|
||||
if (empty($document->getId())) {
|
||||
throw new Exception('Missing ID');
|
||||
}
|
||||
|
||||
try {
|
||||
$new = $projectDB->overwriteDocument($document->getArrayCopy());
|
||||
} catch (\Throwable $th) {
|
||||
var_dump($document);
|
||||
Console::error('Failed to update document: ' . $th->getMessage());
|
||||
continue;
|
||||
}
|
||||
|
||||
if ($new->getId() !== $document->getId()) {
|
||||
throw new Exception('Duplication Error');
|
||||
}
|
||||
}
|
||||
|
||||
$offset = $offset + $limit;
|
||||
}
|
||||
|
||||
$schema = $_SERVER['_APP_DB_SCHEMA'] ?? '';
|
||||
|
||||
try {
|
||||
$statement = $db->prepare("
|
||||
|
||||
CREATE TABLE IF NOT EXISTS `template.database.unique` (
|
||||
`id` int(10) unsigned NOT NULL AUTO_INCREMENT,
|
||||
`key` varchar(128) DEFAULT NULL,
|
||||
PRIMARY KEY (`id`),
|
||||
UNIQUE KEY `index1` (`key`)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS `{$schema}`.`app_{$project->getId()}.database.unique` LIKE `template.database.unique`;
|
||||
ALTER TABLE `{$schema}`.`app_{$project->getId()}.audit.audit` DROP COLUMN IF EXISTS `userType`;
|
||||
ALTER TABLE `{$schema}`.`app_{$project->getId()}.audit.audit` DROP INDEX IF EXISTS `index_1`;
|
||||
ALTER TABLE `{$schema}`.`app_{$project->getId()}.audit.audit` ADD INDEX IF NOT EXISTS `index_1` (`userId` ASC);
|
||||
");
|
||||
|
||||
$statement->closeCursor();
|
||||
|
||||
$statement->execute();
|
||||
} catch (\Exception $e) {
|
||||
Console::error('Failed to alter table for project: ' . $project->getId() . ' with message: ' . $e->getMessage() . '/');
|
||||
}
|
||||
}
|
||||
|
||||
private function fixDocument(Document $document)
|
||||
{
|
||||
$providers = Config::getParam('providers');
|
||||
|
||||
switch ($document->getAttribute('$collection')) {
|
||||
case Database::SYSTEM_COLLECTION_PROJECTS:
|
||||
foreach ($providers as $key => $provider) {
|
||||
if (!empty($document->getAttribute('usersOauth' . \ucfirst($key) . 'Appid'))) {
|
||||
$document
|
||||
->setAttribute('usersOauth2' . \ucfirst($key) . 'Appid', $document->getAttribute('usersOauth' . \ucfirst($key) . 'Appid', ''))
|
||||
->removeAttribute('usersOauth' . \ucfirst($key) . 'Appid');
|
||||
}
|
||||
|
||||
if (!empty($document->getAttribute('usersOauth' . \ucfirst($key) . 'Secret'))) {
|
||||
$document
|
||||
->setAttribute('usersOauth2' . \ucfirst($key) . 'Secret', $document->getAttribute('usersOauth' . \ucfirst($key) . 'Secret', ''))
|
||||
->removeAttribute('usersOauth' . \ucfirst($key) . 'Secret');
|
||||
}
|
||||
}
|
||||
break;
|
||||
|
||||
case Database::SYSTEM_COLLECTION_PROJECTS:
|
||||
case Database::SYSTEM_COLLECTION_TASKS:
|
||||
$document->setAttribute('security', ($document->getAttribute('security')) ? true : false);
|
||||
break;
|
||||
|
||||
case Database::SYSTEM_COLLECTION_USERS:
|
||||
foreach ($providers as $key => $provider) {
|
||||
if (!empty($document->getAttribute('oauth' . \ucfirst($key)))) {
|
||||
$document
|
||||
->setAttribute('oauth2' . \ucfirst($key), $document->getAttribute('oauth' . \ucfirst($key), ''))
|
||||
->removeAttribute('oauth' . \ucfirst($key));
|
||||
}
|
||||
|
||||
if (!empty($document->getAttribute('oauth' . \ucfirst($key) . 'AccessToken'))) {
|
||||
$document
|
||||
->setAttribute('oauth2' . \ucfirst($key) . 'AccessToken', $document->getAttribute('oauth' . \ucfirst($key) . 'AccessToken', ''))
|
||||
->removeAttribute('oauth' . \ucfirst($key) . 'AccessToken');
|
||||
}
|
||||
}
|
||||
|
||||
if ($document->getAttribute('confirm', null) !== null) {
|
||||
$document
|
||||
->setAttribute('emailVerification', $document->getAttribute('confirm', $document->getAttribute('emailVerification', false)))
|
||||
->removeAttribute('confirm');
|
||||
}
|
||||
break;
|
||||
|
||||
case Database::SYSTEM_COLLECTION_PLATFORMS:
|
||||
if ($document->getAttribute('url', null) !== null) {
|
||||
$document
|
||||
->setAttribute('hostname', \parse_url($document->getAttribute('url', $document->getAttribute('hostname', '')), PHP_URL_HOST))
|
||||
->removeAttribute('url');
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
$document
|
||||
->setAttribute('$id', $document->getAttribute('$uid', $document->getAttribute('$id')))
|
||||
->removeAttribute('$uid');
|
||||
|
||||
foreach ($document as &$attr) { // Handle child documents
|
||||
if ($attr instanceof Document) {
|
||||
$attr = $this->fixDocument($attr);
|
||||
}
|
||||
|
||||
if (\is_array($attr)) {
|
||||
foreach ($attr as &$child) {
|
||||
if ($child instanceof Document) {
|
||||
$child = $this->fixDocument($child);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return $document;
|
||||
}
|
||||
}
|
17
src/Appwrite/Migration/Version/V06.php
Normal file
17
src/Appwrite/Migration/Version/V06.php
Normal file
|
@ -0,0 +1,17 @@
|
|||
<?php
|
||||
|
||||
namespace Appwrite\Migration\Version;
|
||||
|
||||
use Utopia\CLI\Console;
|
||||
use Appwrite\Migration\Migration;
|
||||
|
||||
class V06 extends Migration
|
||||
{
|
||||
public function execute(): void
|
||||
{
|
||||
Console::log('I got nothing to do. Yet.');
|
||||
|
||||
//TODO: migrate new `filter` property
|
||||
|
||||
}
|
||||
}
|
Loading…
Reference in a new issue