feat(migratio): add options
This commit is contained in:
Torsten Dittmann
parent
3f92e6991a
commit
9d47280d78
|
|
@ -11,7 +11,7 @@ use Appwrite\Database\Adapter\Redis as RedisAdapter;
|
|||
use Appwrite\Migration\Migration;
|
||||
use Utopia\Validator\Text;
|
||||
|
||||
Config::load('collections.old', __DIR__.'/../config/collections.old.php');
|
||||
Config::load('collections.old', __DIR__ . '/../config/collections.old.php');
|
||||
|
||||
$cli
|
||||
->task('migrate')
|
||||
|
|
@ -23,22 +23,54 @@ $cli
|
|||
Console::exit(1);
|
||||
return;
|
||||
}
|
||||
|
||||
$options = [];
|
||||
if (str_starts_with($version, '0.12.')) {
|
||||
Console::error('WARNING');
|
||||
Console::warning('Migrating to Version 0.12.x introduces a major breaking change within the Database Service!');
|
||||
Console::warning('Before migrating, please read about the breaking changes here:');
|
||||
Console::info('https://appwrite.io/guide-to-db-migration');
|
||||
$confirm = Console::confirm("If you want to proceed, type 'yes':");
|
||||
if($confirm != 'yes') {
|
||||
if ($confirm != 'yes') {
|
||||
Console::exit(1);
|
||||
return;
|
||||
}
|
||||
|
||||
Console::log('');
|
||||
Console::log('Collections');
|
||||
Console::log('--------------------');
|
||||
Console::warning('Be aware that following actions will happen during the migration:');
|
||||
Console::warning('- Nested Document rules will be migrated to String attributes');
|
||||
Console::warning('- Numeric rules will be migrated to float attributes');
|
||||
Console::info("Do you want to migrate your Database Collections?");
|
||||
$options['migrateCollections'] = Console::confirm("Type 'yes' or 'no':");
|
||||
|
||||
if ($options['migrateCollections'] === 'yes') {
|
||||
Console::log('');
|
||||
Console::log('Documents');
|
||||
Console::log('------------------');
|
||||
Console::warning('Be aware that following actions will happen during the migration:');
|
||||
Console::warning('- Nested Documents will be stored as JSON values');
|
||||
Console::warning('- All Numeric values will be converted to float');
|
||||
Console::info("Do you want to migrate your Database Documents?");
|
||||
$options['migrateDocuments'] = Console::confirm("Type 'yes' or 'no':");
|
||||
} else {
|
||||
$options['migrateDocuments'] = 'no';
|
||||
}
|
||||
|
||||
|
||||
if (
|
||||
!in_array($options['migrateDocuments'], ['yes', 'no'])
|
||||
|| !in_array($options['migrateCollections'], ['yes', 'no'])
|
||||
) {
|
||||
Console::error("You must reply with 'yes' or 'no'!");
|
||||
Console::exit(1);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
Config::load('collectionsold' , __DIR__.'/../config/collections.old.php');
|
||||
Config::load('collectionsold', __DIR__ . '/../config/collections.old.php');
|
||||
|
||||
Console::success('Starting Data Migration to version '.$version);
|
||||
Console::success('Starting Data Migration to version ' . $version);
|
||||
|
||||
$db = $register->get('db', true);
|
||||
$cache = $register->get('cache', true);
|
||||
|
|
@ -63,8 +95,8 @@ $cli
|
|||
$projects = [$console];
|
||||
$count = 0;
|
||||
|
||||
$class = 'Appwrite\\Migration\\Version\\'.Migration::$versions[$version];
|
||||
$migration = new $class($register->get('db'), $register->get('cache'));
|
||||
$class = 'Appwrite\\Migration\\Version\\' . Migration::$versions[$version];
|
||||
$migration = new $class($register->get('db'), $register->get('cache'), $options);
|
||||
|
||||
while ($sum > 0) {
|
||||
foreach ($projects as $project) {
|
||||
|
|
@ -74,7 +106,7 @@ $cli
|
|||
->execute();
|
||||
} catch (\Throwable $th) {
|
||||
throw $th;
|
||||
Console::error('Failed to update project ("'.$project->getId().'") version with error: '.$th->getMessage());
|
||||
Console::error('Failed to update project ("' . $project->getId() . '") version with error: ' . $th->getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -82,7 +114,7 @@ $cli
|
|||
'limit' => $limit,
|
||||
'offset' => $offset,
|
||||
'filters' => [
|
||||
'$collection='.Database::SYSTEM_COLLECTION_PROJECTS,
|
||||
'$collection=' . Database::SYSTEM_COLLECTION_PROJECTS,
|
||||
],
|
||||
]);
|
||||
|
||||
|
|
@ -91,7 +123,7 @@ $cli
|
|||
$count = $count + $sum;
|
||||
|
||||
if ($sum > 0) {
|
||||
Console::log('Fetched '.$count.'/'.$consoleDB->getSum().' projects...');
|
||||
Console::log('Fetched ' . $count . '/' . $consoleDB->getSum() . ' projects...');
|
||||
}
|
||||
}
|
||||
$cache->flushAll();
|
||||
|
|
|
|||
|
|
@ -12,6 +12,11 @@ use Utopia\Exception;
|
|||
|
||||
abstract class Migration
|
||||
{
|
||||
/**
|
||||
* @var array
|
||||
*/
|
||||
protected array $options;
|
||||
|
||||
/**
|
||||
* @var PDO
|
||||
*/
|
||||
|
|
@ -68,8 +73,9 @@ abstract class Migration
|
|||
*
|
||||
* @param PDO $pdo
|
||||
*/
|
||||
public function __construct(PDO $db, Redis $cache = null)
|
||||
public function __construct(PDO $db, Redis $cache = null, array $options = [])
|
||||
{
|
||||
$this->options = $options;
|
||||
$this->db = $db;
|
||||
if(!is_null($cache)) {
|
||||
$this->cache = $cache;
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ use Exception;
|
|||
use PDO;
|
||||
use Redis;
|
||||
use Swoole\Runtime;
|
||||
use Throwable;
|
||||
use Utopia\Abuse\Adapters\TimeLimit;
|
||||
use Utopia\Audit\Audit;
|
||||
use Utopia\Cache\Cache;
|
||||
|
|
@ -18,6 +19,9 @@ use Utopia\Config\Config;
|
|||
use Utopia\Database\Adapter\MariaDB;
|
||||
use Utopia\Database\Database;
|
||||
use Utopia\Database\Document;
|
||||
use Utopia\Database\Exception\Limit;
|
||||
use Utopia\Database\Exception\Authorization as ExceptionAuthorization;
|
||||
use Utopia\Database\Exception\Structure;
|
||||
use Utopia\Database\Query;
|
||||
use Utopia\Database\Validator\Authorization;
|
||||
|
||||
|
|
@ -32,9 +36,10 @@ class V11 extends Migration
|
|||
protected array $oldCollections;
|
||||
protected array $newCollections;
|
||||
|
||||
public function __construct(PDO $db, Redis $cache = null)
|
||||
public function __construct(PDO $db, Redis $cache = null, array $options = [])
|
||||
{
|
||||
parent::__construct($db, $cache);
|
||||
parent::__construct($db, $cache, $options);
|
||||
$this->options = array_map(fn($option) => $option === 'yes' ? true : false, $this->options);
|
||||
|
||||
if (!is_null($cache)) {
|
||||
$cacheAdapter = new Cache(new RedisCache($this->cache));
|
||||
|
|
@ -151,163 +156,8 @@ class V11 extends Migration
|
|||
|
||||
$this->dbInternal->createCollection($key, $attributes, $indexes);
|
||||
}
|
||||
|
||||
$sum = $this->limit;
|
||||
$offset = 0;
|
||||
|
||||
/**
|
||||
* Migrate external collections for Project
|
||||
*/
|
||||
while ($sum >= $this->limit) {
|
||||
$databaseCollections = $this->oldProjectDB->getCollection([
|
||||
'limit' => $this->limit,
|
||||
'offset' => $offset,
|
||||
'orderType' => 'DESC',
|
||||
'filters' => [
|
||||
'$collection=' . OldDatabase::SYSTEM_COLLECTION_COLLECTIONS,
|
||||
]
|
||||
]);
|
||||
|
||||
$sum = \count($databaseCollections);
|
||||
Console::log('Migrating Collections: ' . $offset . ' / ' . $this->oldProjectDB->getSum());
|
||||
|
||||
foreach ($databaseCollections as $oldCollection) {
|
||||
$id = $oldCollection->getId();
|
||||
$permissions = $oldCollection->getPermissions();
|
||||
$name = $oldCollection->getAttribute('name');
|
||||
$newCollection = $this->dbExternal->getCollection($id);
|
||||
|
||||
if ($newCollection->isEmpty()) {
|
||||
$this->dbExternal->createCollection($id);
|
||||
/**
|
||||
* Migrate permissions
|
||||
*/
|
||||
$read = $this->migrateWildcardPermissions($permissions['read'] ?? []);
|
||||
$write = $this->migrateWildcardPermissions($permissions['write'] ?? []);
|
||||
|
||||
/**
|
||||
* Suffix collection name with a subsequent number to make it unique if possible.
|
||||
*/
|
||||
$suffix = 1;
|
||||
while ($this->dbInternal->findOne('collections', [
|
||||
new Query('name', Query::TYPE_EQUAL, [$name])
|
||||
])) {
|
||||
$name .= ' - ' . $suffix++;
|
||||
}
|
||||
|
||||
$this->dbInternal->createDocument('collections', new Document([
|
||||
'$id' => $id,
|
||||
'$read' => $read,
|
||||
'$write' => $write,
|
||||
'permission' => 'document',
|
||||
'dateCreated' => time(),
|
||||
'dateUpdated' => time(),
|
||||
'name' => $name,
|
||||
'search' => implode(' ', [$id, $name]),
|
||||
]));
|
||||
} else {
|
||||
Console::warning('Skipped Collection ' . $newCollection->getId() . ' from ' . $newCollection->getCollection());
|
||||
}
|
||||
|
||||
/**
|
||||
* Migrate collection rules to attributes
|
||||
*/
|
||||
$attributes = $this->getCollectionAttributes($oldCollection);
|
||||
|
||||
foreach ($attributes as $attribute) {
|
||||
try {
|
||||
$this->dbExternal->createAttribute(
|
||||
collection: $attribute['$collection'],
|
||||
id: $attribute['$id'],
|
||||
type: $attribute['type'],
|
||||
size: $attribute['size'],
|
||||
required: $attribute['required'],
|
||||
default: $attribute['default'],
|
||||
signed: $attribute['signed'],
|
||||
array: $attribute['array'],
|
||||
format: null,
|
||||
filters: $attribute['filters']
|
||||
);
|
||||
|
||||
$this->dbInternal->createDocument('attributes', new Document([
|
||||
'$id' => $attribute['$collection'] . '_' . $attribute['$id'],
|
||||
'key' => $attribute['$id'],
|
||||
'collectionId' => $attribute['$collection'],
|
||||
'type' => $attribute['type'],
|
||||
'status' => 'available',
|
||||
'size' => $attribute['size'],
|
||||
'required' => $attribute['required'],
|
||||
'signed' => $attribute['signed'],
|
||||
'default' => $attribute['default'],
|
||||
'array' => $attribute['array'],
|
||||
'format' => null,
|
||||
'filters' => $attribute['filters']
|
||||
]));
|
||||
|
||||
Console::log('Created "' . $attribute['$id'] . '" attribute in collection: ' . $name);
|
||||
} catch (\Throwable $th) {
|
||||
Console::log($th->getMessage() . ' - (' . $attribute['$id'] . '" attribute in collection ' . $name . ')');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Migrate all external documents
|
||||
*/
|
||||
$sumDocs = $this->limit;
|
||||
$offsetDocs = 0;
|
||||
while ($sumDocs >= $this->limit) {
|
||||
$allDocs = $this->oldProjectDB->getCollection([
|
||||
'limit' => $this->limit,
|
||||
'offset' => $offsetDocs,
|
||||
'orderType' => 'DESC',
|
||||
'filters' => [
|
||||
'$collection=' . $id
|
||||
]
|
||||
]);
|
||||
|
||||
$sumDocs = \count($allDocs);
|
||||
foreach ($allDocs as $document) {
|
||||
if (!$this->dbExternal->getDocument($id, $document->getId())->isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
foreach ($document as $key => $attr) {
|
||||
/**
|
||||
* Convert nested Document to JSON strings.
|
||||
*/
|
||||
if ($document->getAttribute($key) instanceof OldDocument) {
|
||||
$document[$key] = json_encode($this->fixDocument($attr)->getArrayCopy());
|
||||
}
|
||||
/**
|
||||
* Convert numeric Attributes to float.
|
||||
*/
|
||||
if (is_numeric($attr)) {
|
||||
$document[$key] = floatval($attr);
|
||||
}
|
||||
if (\is_array($attr)) {
|
||||
foreach ($attr as $index => $child) {
|
||||
/**
|
||||
* Convert array of nested Document to array JSON strings.
|
||||
*/
|
||||
if ($document->getAttribute($key)[$index] instanceof OldDocument) {
|
||||
$document[$key][$index] = json_encode($this->fixDocument($child)->getArrayCopy());
|
||||
}
|
||||
/**
|
||||
* Convert array of numeric Attributes to array float.
|
||||
*/
|
||||
if (is_numeric($attr)) {
|
||||
$document[$key][$index] = floatval($child); // Convert any numeric to float
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
$document = new Document($document->getArrayCopy());
|
||||
$document = $this->migratePermissions($document);
|
||||
$this->dbExternal->createDocument($id, $document);
|
||||
}
|
||||
$offsetDocs += $this->limit;
|
||||
}
|
||||
}
|
||||
$offset += $this->limit;
|
||||
if ($this->options['migrateCollections']) {
|
||||
$this->migrateExternalCollections();
|
||||
}
|
||||
} else {
|
||||
Console::log('Skipped console project migration.');
|
||||
|
|
@ -315,7 +165,7 @@ class V11 extends Migration
|
|||
|
||||
$sum = $this->limit;
|
||||
$offset = 0;
|
||||
Authorization::disable();
|
||||
|
||||
/**
|
||||
* Migrate internal documents
|
||||
*/
|
||||
|
|
@ -373,6 +223,192 @@ class V11 extends Migration
|
|||
Console::log('Migrated ' . $sum . ' Documents.');
|
||||
}
|
||||
|
||||
/**
|
||||
* Migrate external collections for Project
|
||||
*
|
||||
* @return void
|
||||
* @throws Exception
|
||||
* @throws Throwable
|
||||
* @throws Limit
|
||||
* @throws ExceptionAuthorization
|
||||
* @throws Structure
|
||||
*/
|
||||
protected function migrateExternalCollections(): void
|
||||
{
|
||||
$sum = $this->limit;
|
||||
$offset = 0;
|
||||
|
||||
while ($sum >= $this->limit) {
|
||||
$databaseCollections = $this->oldProjectDB->getCollection([
|
||||
'limit' => $this->limit,
|
||||
'offset' => $offset,
|
||||
'orderType' => 'DESC',
|
||||
'filters' => [
|
||||
'$collection=' . OldDatabase::SYSTEM_COLLECTION_COLLECTIONS,
|
||||
]
|
||||
]);
|
||||
|
||||
$sum = \count($databaseCollections);
|
||||
Console::log('Migrating Collections: ' . $offset . ' / ' . $this->oldProjectDB->getSum());
|
||||
|
||||
foreach ($databaseCollections as $oldCollection) {
|
||||
$id = $oldCollection->getId();
|
||||
$permissions = $oldCollection->getPermissions();
|
||||
$name = $oldCollection->getAttribute('name');
|
||||
$newCollection = $this->dbExternal->getCollection($id);
|
||||
|
||||
if ($newCollection->isEmpty()) {
|
||||
$this->dbExternal->createCollection($id);
|
||||
/**
|
||||
* Migrate permissions
|
||||
*/
|
||||
$read = $this->migrateWildcardPermissions($permissions['read'] ?? []);
|
||||
$write = $this->migrateWildcardPermissions($permissions['write'] ?? []);
|
||||
|
||||
/**
|
||||
* Suffix collection name with a subsequent number to make it unique if possible.
|
||||
*/
|
||||
$suffix = 1;
|
||||
while ($this->dbInternal->findOne('collections', [
|
||||
new Query('name', Query::TYPE_EQUAL, [$name])
|
||||
])) {
|
||||
$name .= ' - ' . $suffix++;
|
||||
}
|
||||
|
||||
$this->dbInternal->createDocument('collections', new Document([
|
||||
'$id' => $id,
|
||||
'$read' => $read,
|
||||
'$write' => $write,
|
||||
'permission' => 'document',
|
||||
'dateCreated' => time(),
|
||||
'dateUpdated' => time(),
|
||||
'name' => $name,
|
||||
'search' => implode(' ', [$id, $name]),
|
||||
]));
|
||||
} else {
|
||||
Console::warning('Skipped Collection ' . $newCollection->getId() . ' from ' . $newCollection->getCollection());
|
||||
}
|
||||
|
||||
/**
|
||||
* Migrate collection rules to attributes
|
||||
*/
|
||||
$attributes = $this->getCollectionAttributes($oldCollection);
|
||||
|
||||
foreach ($attributes as $attribute) {
|
||||
try {
|
||||
$this->dbExternal->createAttribute(
|
||||
collection: $attribute['$collection'],
|
||||
id: $attribute['$id'],
|
||||
type: $attribute['type'],
|
||||
size: $attribute['size'],
|
||||
required: $attribute['required'],
|
||||
default: $attribute['default'],
|
||||
signed: $attribute['signed'],
|
||||
array: $attribute['array'],
|
||||
format: null,
|
||||
filters: $attribute['filters']
|
||||
);
|
||||
|
||||
$this->dbInternal->createDocument('attributes', new Document([
|
||||
'$id' => $attribute['$collection'] . '_' . $attribute['$id'],
|
||||
'key' => $attribute['$id'],
|
||||
'collectionId' => $attribute['$collection'],
|
||||
'type' => $attribute['type'],
|
||||
'status' => 'available',
|
||||
'size' => $attribute['size'],
|
||||
'required' => $attribute['required'],
|
||||
'signed' => $attribute['signed'],
|
||||
'default' => $attribute['default'],
|
||||
'array' => $attribute['array'],
|
||||
'format' => null,
|
||||
'filters' => $attribute['filters']
|
||||
]));
|
||||
|
||||
Console::log('Created "' . $attribute['$id'] . '" attribute in collection: ' . $name);
|
||||
} catch (\Throwable $th) {
|
||||
Console::log($th->getMessage() . ' - (' . $attribute['$id'] . '" attribute in collection ' . $name . ')');
|
||||
}
|
||||
}
|
||||
if ($this->options['migrateDocuments']) {
|
||||
$this->migrateExternalDocuments(collection: $id);
|
||||
}
|
||||
}
|
||||
$offset += $this->limit;
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Migrate all external documents
|
||||
*
|
||||
* @return void
|
||||
* @throws Exception
|
||||
* @throws Throwable
|
||||
* @throws ExceptionAuthorization
|
||||
* @throws Structure
|
||||
*/
|
||||
protected function migrateExternalDocuments(string $collection): void
|
||||
{
|
||||
$sum = $this->limit;
|
||||
$offset = 0;
|
||||
while ($sum >= $this->limit) {
|
||||
$allDocs = $this->oldProjectDB->getCollection([
|
||||
'limit' => $this->limit,
|
||||
'offset' => $offset,
|
||||
'orderType' => 'DESC',
|
||||
'filters' => [
|
||||
'$collection=' . $collection
|
||||
]
|
||||
]);
|
||||
|
||||
$sum = \count($allDocs);
|
||||
foreach ($allDocs as $document) {
|
||||
if (!$this->dbExternal->getDocument($collection, $document->getId())->isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
foreach ($document as $key => $attr) {
|
||||
/**
|
||||
* Convert nested Document to JSON strings.
|
||||
*/
|
||||
if ($document->getAttribute($key) instanceof OldDocument) {
|
||||
$document[$key] = json_encode($this->fixDocument($attr)->getArrayCopy());
|
||||
}
|
||||
/**
|
||||
* Convert numeric Attributes to float.
|
||||
*/
|
||||
if (is_numeric($attr)) {
|
||||
$document[$key] = floatval($attr);
|
||||
}
|
||||
if (\is_array($attr)) {
|
||||
foreach ($attr as $index => $child) {
|
||||
/**
|
||||
* Convert array of nested Document to array JSON strings.
|
||||
*/
|
||||
if ($document->getAttribute($key)[$index] instanceof OldDocument) {
|
||||
$document[$key][$index] = json_encode($this->fixDocument($child)->getArrayCopy());
|
||||
}
|
||||
/**
|
||||
* Convert array of numeric Attributes to array float.
|
||||
*/
|
||||
if (is_numeric($attr)) {
|
||||
$document[$key][$index] = floatval($child); // Convert any numeric to float
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
$document = new Document($document->getArrayCopy());
|
||||
$document = $this->migratePermissions($document);
|
||||
$this->dbExternal->createDocument($collection, $document);
|
||||
}
|
||||
$offset += $this->limit;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Migrates single docuemnt.
|
||||
*
|
||||
* @param OldDocument $oldDocument
|
||||
* @return Document
|
||||
* @throws Exception
|
||||
*/
|
||||
protected function fixDocument(OldDocument $oldDocument): Document
|
||||
{
|
||||
$document = new Document($oldDocument->getArrayCopy());
|
||||
|
|
@ -620,7 +656,6 @@ class V11 extends Migration
|
|||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param Document $document
|
||||
* @return string|null
|
||||
* @throws Exception
|
||||
|
|
|
|||
Loading…
Reference in a new issue