Merge pull request #2474 from appwrite/feat-database-indexing-migration
feat(tasks): migration for 0.12.x
This commit is contained in:
commit
166ebf7225
|
@ -1617,8 +1617,8 @@ foreach ($providers as $index => $provider) {
|
|||
foreach ($auth as $index => $method) {
|
||||
$collections[Database::SYSTEM_COLLECTION_PROJECTS]['rules'][] = [
|
||||
'$collection' => Database::SYSTEM_COLLECTION_RULES,
|
||||
'label' => $method['name'] || '',
|
||||
'key' => $method['key'] || '',
|
||||
'label' => $method['name'] ?? '',
|
||||
'key' => $method['key'] ?? '',
|
||||
'type' => Database::SYSTEM_VAR_TYPE_BOOLEAN,
|
||||
'default' => true,
|
||||
'required' => false,
|
||||
|
|
|
@ -487,7 +487,7 @@ $collections = [
|
|||
'size' => 16384,
|
||||
'signed' => true,
|
||||
'required' => false,
|
||||
'default' => null,
|
||||
'default' => [],
|
||||
'array' => false,
|
||||
'filters' => ['json'],
|
||||
],
|
||||
|
@ -498,7 +498,7 @@ $collections = [
|
|||
'size' => 16384,
|
||||
'signed' => true,
|
||||
'required' => false,
|
||||
'default' => null,
|
||||
'default' => [],
|
||||
'array' => false,
|
||||
'filters' => ['json'],
|
||||
],
|
||||
|
@ -509,7 +509,7 @@ $collections = [
|
|||
'size' => 16384,
|
||||
'signed' => true,
|
||||
'required' => false,
|
||||
'default' => null,
|
||||
'default' => [],
|
||||
'array' => false,
|
||||
'filters' => ['json', 'encrypt'],
|
||||
],
|
||||
|
@ -884,7 +884,7 @@ $collections = [
|
|||
'format' => '',
|
||||
'size' => Database::LENGTH_KEY,
|
||||
'signed' => true,
|
||||
'required' => true,
|
||||
'required' => false,
|
||||
'default' => null,
|
||||
'array' => false,
|
||||
'filters' => [],
|
||||
|
@ -895,7 +895,7 @@ $collections = [
|
|||
'format' => '',
|
||||
'size' => Database::LENGTH_KEY, // TODO will the length suffice after encryption?
|
||||
'signed' => true,
|
||||
'required' => true,
|
||||
'required' => false,
|
||||
'default' => null,
|
||||
'array' => false,
|
||||
'filters' => ['encrypt'],
|
||||
|
@ -1001,7 +1001,7 @@ $collections = [
|
|||
'size' => 16384,
|
||||
'signed' => true,
|
||||
'required' => false,
|
||||
'default' => null,
|
||||
'default' => [],
|
||||
'array' => false,
|
||||
'filters' => ['json'],
|
||||
],
|
||||
|
@ -1045,7 +1045,7 @@ $collections = [
|
|||
'size' => 16384,
|
||||
'signed' => true,
|
||||
'required' => false,
|
||||
'default' => null,
|
||||
'default' => [],
|
||||
'array' => true,
|
||||
'filters' => ['json'],
|
||||
],
|
||||
|
@ -1056,7 +1056,7 @@ $collections = [
|
|||
'size' => 16384,
|
||||
'signed' => true,
|
||||
'required' => false,
|
||||
'default' => null,
|
||||
'default' => [],
|
||||
'array' => true,
|
||||
'filters' => ['json'],
|
||||
],
|
||||
|
@ -1067,7 +1067,7 @@ $collections = [
|
|||
'size' => 16384,
|
||||
'signed' => true,
|
||||
'required' => false,
|
||||
'default' => null,
|
||||
'default' => [],
|
||||
'array' => true,
|
||||
'filters' => ['json'],
|
||||
],
|
||||
|
@ -1816,7 +1816,7 @@ $collections = [
|
|||
'size' => 8192,
|
||||
'signed' => true,
|
||||
'required' => false,
|
||||
'default' => null,
|
||||
'default' => [],
|
||||
'array' => false,
|
||||
'filters' => ['json', 'encrypt'],
|
||||
],
|
||||
|
|
42
app/init.php
42
app/init.php
|
@ -21,6 +21,7 @@ use Appwrite\Extend\PDO;
|
|||
use Ahc\Jwt\JWT;
|
||||
use Ahc\Jwt\JWTException;
|
||||
use Appwrite\Auth\Auth;
|
||||
use Appwrite\Database\Database as DatabaseOld;
|
||||
use Appwrite\Event\Event;
|
||||
use Appwrite\Network\Validator\Email;
|
||||
use Appwrite\Network\Validator\IP;
|
||||
|
@ -153,6 +154,43 @@ if(!empty($user) || !empty($pass)) {
|
|||
Resque::setBackend(App::getEnv('_APP_REDIS_HOST', '').':'.App::getEnv('_APP_REDIS_PORT', ''));
|
||||
}
|
||||
|
||||
/**
|
||||
* Old DB Filters
|
||||
*/
|
||||
DatabaseOld::addFilter('json',
|
||||
function($value) {
|
||||
if(!is_array($value)) {
|
||||
return $value;
|
||||
}
|
||||
return json_encode($value);
|
||||
},
|
||||
function($value) {
|
||||
return json_decode($value, true);
|
||||
}
|
||||
);
|
||||
|
||||
DatabaseOld::addFilter('encrypt',
|
||||
function($value) {
|
||||
$key = App::getEnv('_APP_OPENSSL_KEY_V1');
|
||||
$iv = OpenSSL::randomPseudoBytes(OpenSSL::cipherIVLength(OpenSSL::CIPHER_AES_128_GCM));
|
||||
$tag = null;
|
||||
|
||||
return json_encode([
|
||||
'data' => OpenSSL::encrypt($value, OpenSSL::CIPHER_AES_128_GCM, $key, 0, $iv, $tag),
|
||||
'method' => OpenSSL::CIPHER_AES_128_GCM,
|
||||
'iv' => bin2hex($iv),
|
||||
'tag' => bin2hex($tag),
|
||||
'version' => '1',
|
||||
]);
|
||||
},
|
||||
function($value) {
|
||||
$value = json_decode($value, true);
|
||||
$key = App::getEnv('_APP_OPENSSL_KEY_V'.$value['version']);
|
||||
|
||||
return OpenSSL::decrypt($value['data'], $value['method'], $key, 0, hex2bin($value['iv']), hex2bin($value['tag']));
|
||||
}
|
||||
);
|
||||
|
||||
/**
|
||||
* New DB Filters
|
||||
*/
|
||||
|
@ -176,7 +214,7 @@ Database::addFilter('enum',
|
|||
return $value;
|
||||
},
|
||||
function($value, Document $attribute) {
|
||||
$formatOptions = json_decode($attribute->getAttribute('formatOptions', []), true);
|
||||
$formatOptions = json_decode($attribute->getAttribute('formatOptions', '[]'), true);
|
||||
if (isset($formatOptions['elements'])) {
|
||||
$attribute->setAttribute('elements', $formatOptions['elements']);
|
||||
}
|
||||
|
@ -195,7 +233,7 @@ Database::addFilter('range',
|
|||
return $value;
|
||||
},
|
||||
function($value, Document $attribute) {
|
||||
$formatOptions = json_decode($attribute->getAttribute('formatOptions', []), true);
|
||||
$formatOptions = json_decode($attribute->getAttribute('formatOptions', '[]'), true);
|
||||
if (isset($formatOptions['min']) || isset($formatOptions['max'])) {
|
||||
$attribute
|
||||
->setAttribute('min', $formatOptions['min'])
|
||||
|
|
|
@ -11,55 +11,106 @@ use Appwrite\Database\Adapter\Redis as RedisAdapter;
|
|||
use Appwrite\Migration\Migration;
|
||||
use Utopia\Validator\Text;
|
||||
|
||||
Config::load('collections.old', __DIR__.'/../config/collections.old.php');
|
||||
Config::load('collections.old', __DIR__ . '/../config/collections.old.php');
|
||||
|
||||
$cli
|
||||
->task('migrate')
|
||||
->param('version', APP_VERSION_STABLE, new Text(8), 'Version to migrate to.', true)
|
||||
->action(function ($version) use ($register) {
|
||||
Authorization::disable();
|
||||
if (!array_key_exists($version, Migration::$versions)) {
|
||||
Console::error("Version {$version} not found.");
|
||||
Console::exit(1);
|
||||
return;
|
||||
}
|
||||
$options = [];
|
||||
if (str_starts_with($version, '0.12.')) {
|
||||
Console::error('--------------------');
|
||||
Console::error('WARNING');
|
||||
Console::error('--------------------');
|
||||
Console::warning('Migrating to Version 0.12.x introduces a major breaking change within the Database Service!');
|
||||
Console::warning('Before migrating, please read about the breaking changes here:');
|
||||
Console::info('https://appwrite.io/guide-to-db-migration');
|
||||
$confirm = Console::confirm("If you want to proceed, type 'yes':");
|
||||
if ($confirm != 'yes') {
|
||||
Console::exit(1);
|
||||
return;
|
||||
}
|
||||
|
||||
Console::log('');
|
||||
Console::log('Collections');
|
||||
Console::log('--------------------');
|
||||
Console::warning('Be aware that following actions will happen during the migration:');
|
||||
Console::warning('- Nested Document rules will be migrated to String attributes');
|
||||
Console::warning('- Numeric rules will be migrated to float attributes');
|
||||
Console::warning('- Wildcard and Markdown rules will be converted to string attributes');
|
||||
Console::info("Do you want to migrate your Database Collections?");
|
||||
$options['migrateCollections'] = Console::confirm("Type 'yes' or 'no':");
|
||||
|
||||
if ($options['migrateCollections'] === 'yes') {
|
||||
Console::log('');
|
||||
Console::log('Documents');
|
||||
Console::log('------------------');
|
||||
Console::warning('Be aware that following actions will happen during the migration:');
|
||||
Console::warning('- Nested Documents will be stored as JSON values');
|
||||
Console::warning('- All Numeric values will be converted to float');
|
||||
Console::warning('- All Wildcard and Markdown values will be converted to string');
|
||||
Console::info("Do you want to migrate your Database Documents?");
|
||||
$options['migrateDocuments'] = Console::confirm("Type 'yes' or 'no':");
|
||||
} else {
|
||||
$options['migrateDocuments'] = 'no';
|
||||
}
|
||||
|
||||
|
||||
if (
|
||||
!in_array($options['migrateDocuments'], ['yes', 'no'])
|
||||
|| !in_array($options['migrateCollections'], ['yes', 'no'])
|
||||
) {
|
||||
Console::error("You must reply with 'yes' or 'no'!");
|
||||
Console::exit(1);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
Config::load('collectionsold', __DIR__ . '/../config/collections.old.php');
|
||||
|
||||
Console::success('Starting Data Migration to version ' . $version);
|
||||
|
||||
Console::success('Starting Data Migration to version '.$version);
|
||||
$db = $register->get('db', true);
|
||||
$cache = $register->get('cache', true);
|
||||
$cache->flushAll();
|
||||
|
||||
$consoleDB = new Database();
|
||||
$consoleDB
|
||||
->setAdapter(new RedisAdapter(new MySQLAdapter($db, $cache), $cache))
|
||||
->setNamespace('app_console') // Main DB
|
||||
->setMocks(Config::getParam('collections.old', []));
|
||||
->setMocks(Config::getParam('collectionsold', []));
|
||||
|
||||
$projectDB = new Database();
|
||||
$projectDB
|
||||
->setAdapter(new RedisAdapter(new MySQLAdapter($db, $cache), $cache))
|
||||
->setMocks(Config::getParam('collections.old', []));
|
||||
->setMocks(Config::getParam('collectionsold', []));
|
||||
|
||||
$console = $consoleDB->getDocument('console');
|
||||
|
||||
Authorization::disable();
|
||||
|
||||
$limit = 30;
|
||||
$sum = 30;
|
||||
$offset = 0;
|
||||
$projects = [$console];
|
||||
$count = 0;
|
||||
|
||||
$class = 'Appwrite\\Migration\\Version\\'.Migration::$versions[$version];
|
||||
$migration = new $class($register->get('db'));
|
||||
$class = 'Appwrite\\Migration\\Version\\' . Migration::$versions[$version];
|
||||
$migration = new $class($register->get('db'), $register->get('cache'), $options);
|
||||
|
||||
while ($sum > 0) {
|
||||
foreach ($projects as $project) {
|
||||
try {
|
||||
$migration
|
||||
->setProject($project, $projectDB)
|
||||
->setProject($project, $projectDB, $consoleDB)
|
||||
->execute();
|
||||
} catch (\Throwable $th) {
|
||||
throw $th;
|
||||
Console::error('Failed to update project ("'.$project->getId().'") version with error: '.$th->getMessage());
|
||||
Console::error('Failed to update project ("' . $project->getId() . '") version with error: ' . $th->getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -67,7 +118,7 @@ $cli
|
|||
'limit' => $limit,
|
||||
'offset' => $offset,
|
||||
'filters' => [
|
||||
'$collection='.Database::SYSTEM_COLLECTION_PROJECTS,
|
||||
'$collection=' . Database::SYSTEM_COLLECTION_PROJECTS,
|
||||
],
|
||||
]);
|
||||
|
||||
|
@ -76,9 +127,11 @@ $cli
|
|||
$count = $count + $sum;
|
||||
|
||||
if ($sum > 0) {
|
||||
Console::log('Fetched '.$count.'/'.$consoleDB->getSum().' projects...');
|
||||
Console::log('Fetched ' . $count . '/' . $consoleDB->getSum() . ' projects...');
|
||||
}
|
||||
}
|
||||
$cache->flushAll();
|
||||
|
||||
Swoole\Event::wait(); // Wait for Coroutines to finish
|
||||
Console::success('Data Migration Completed');
|
||||
});
|
||||
|
|
|
@ -91,6 +91,26 @@ class Document extends ArrayObject
|
|||
return $temp;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get Document Attributes
|
||||
*
|
||||
* @return array
|
||||
*/
|
||||
public function getAttributes(): array
|
||||
{
|
||||
$attributes = [];
|
||||
|
||||
foreach ($this as $attribute => $value) {
|
||||
if(array_key_exists($attribute, ['$id' => true, '$permissions' => true, '$collection' => true, '$execute' => []])) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$attributes[$attribute] = $value;
|
||||
}
|
||||
|
||||
return $attributes;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set Attribute.
|
||||
*
|
||||
|
@ -215,7 +235,7 @@ class Document extends ArrayObject
|
|||
*
|
||||
* @return array
|
||||
*/
|
||||
public function getArrayCopy(array $whitelist = [], array $blacklist = [])
|
||||
public function getArrayCopy(array $whitelist = [], array $blacklist = []): array
|
||||
{
|
||||
$array = parent::getArrayCopy();
|
||||
|
||||
|
|
|
@ -2,34 +2,50 @@
|
|||
|
||||
namespace Appwrite\Migration;
|
||||
|
||||
use Appwrite\Database\Document;
|
||||
use Appwrite\Database\Database;
|
||||
use Appwrite\Database\Document as OldDocument;
|
||||
use Appwrite\Database\Database as OldDatabase;
|
||||
use PDO;
|
||||
use Redis;
|
||||
use Swoole\Runtime;
|
||||
use Utopia\CLI\Console;
|
||||
use Utopia\Exception;
|
||||
|
||||
abstract class Migration
|
||||
{
|
||||
/**
|
||||
* @var array
|
||||
*/
|
||||
protected array $options;
|
||||
|
||||
/**
|
||||
* @var PDO
|
||||
*/
|
||||
protected $db;
|
||||
protected PDO $db;
|
||||
|
||||
/**
|
||||
* @var Redis
|
||||
*/
|
||||
protected Redis $cache;
|
||||
|
||||
/**
|
||||
* @var int
|
||||
*/
|
||||
protected $limit = 50;
|
||||
protected int $limit = 500;
|
||||
|
||||
/**
|
||||
* @var Document
|
||||
* @var OldDocument
|
||||
*/
|
||||
protected $project;
|
||||
protected OldDocument $project;
|
||||
|
||||
/**
|
||||
* @var Database
|
||||
* @var OldDatabase
|
||||
*/
|
||||
protected $projectDB;
|
||||
protected OldDatabase $oldProjectDB;
|
||||
|
||||
/**
|
||||
* @var OldDatabase
|
||||
*/
|
||||
protected OldDatabase $oldConsoleDB;
|
||||
|
||||
/**
|
||||
* @var array
|
||||
|
@ -49,32 +65,44 @@ abstract class Migration
|
|||
'0.10.3' => 'V09',
|
||||
'0.10.4' => 'V09',
|
||||
'0.11.0' => 'V10',
|
||||
'0.12.0' => 'V10',
|
||||
'0.12.0' => 'V11',
|
||||
];
|
||||
|
||||
/**
|
||||
* Migration constructor.
|
||||
*
|
||||
* @param PDO $pdo
|
||||
* @param PDO $db
|
||||
* @param Redis|null $cache
|
||||
* @param array $options
|
||||
* @return void
|
||||
*/
|
||||
public function __construct(PDO $db)
|
||||
public function __construct(PDO $db, Redis $cache = null, array $options = [])
|
||||
{
|
||||
$this->options = $options;
|
||||
$this->db = $db;
|
||||
if (!is_null($cache)) {
|
||||
$this->cache = $cache;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Set project for migration.
|
||||
*
|
||||
* @param Document $project
|
||||
* @param Database $projectDB
|
||||
* @param OldDocument $project
|
||||
* @param OldDatabase $projectDB
|
||||
* @param OldDatabase $oldConsoleDB
|
||||
*
|
||||
* @return Migration
|
||||
* @return self
|
||||
*/
|
||||
public function setProject(Document $project, Database $projectDB): Migration
|
||||
public function setProject(OldDocument $project, OldDatabase $projectDB, OldDatabase $oldConsoleDB): self
|
||||
{
|
||||
$this->project = $project;
|
||||
$this->projectDB = $projectDB;
|
||||
$this->projectDB->setNamespace('app_' . $project->getId());
|
||||
|
||||
$this->oldProjectDB = $projectDB;
|
||||
$this->oldProjectDB->setNamespace('app_' . $project->getId());
|
||||
|
||||
$this->oldConsoleDB = $oldConsoleDB;
|
||||
|
||||
return $this;
|
||||
}
|
||||
|
||||
|
@ -117,7 +145,7 @@ abstract class Migration
|
|||
}
|
||||
|
||||
try {
|
||||
$new = $this->projectDB->overwriteDocument($document->getArrayCopy());
|
||||
$new = $this->projectDB->overwriteDocument($new->getArrayCopy());
|
||||
} catch (\Throwable $th) {
|
||||
Console::error('Failed to update document: ' . $th->getMessage());
|
||||
return;
|
||||
|
@ -134,7 +162,14 @@ abstract class Migration
|
|||
}
|
||||
}
|
||||
|
||||
public function check_diff_multi($array1, $array2)
|
||||
/**
|
||||
* Checks 2 arrays for differences.
|
||||
*
|
||||
* @param array $array1
|
||||
* @param array $array2
|
||||
* @return array
|
||||
*/
|
||||
public function check_diff_multi(array $array1, array $array2): array
|
||||
{
|
||||
$result = array();
|
||||
|
||||
|
|
683
src/Appwrite/Migration/Version/V11.php
Normal file
683
src/Appwrite/Migration/Version/V11.php
Normal file
|
@ -0,0 +1,683 @@
|
|||
<?php
|
||||
|
||||
namespace Appwrite\Migration\Version;
|
||||
|
||||
use Appwrite\Database\Database as OldDatabase;
|
||||
use Appwrite\Database\Document as OldDocument;
|
||||
use Appwrite\Migration\Migration;
|
||||
use Exception;
|
||||
use PDO;
|
||||
use Redis;
|
||||
use Swoole\Runtime;
|
||||
use Throwable;
|
||||
use Utopia\Abuse\Adapters\TimeLimit;
|
||||
use Utopia\Audit\Audit;
|
||||
use Utopia\Cache\Cache;
|
||||
use Utopia\CLI\Console;
|
||||
use Utopia\Cache\Adapter\Redis as RedisCache;
|
||||
use Utopia\Config\Config;
|
||||
use Utopia\Database\Adapter\MariaDB;
|
||||
use Utopia\Database\Database;
|
||||
use Utopia\Database\Document;
|
||||
use Utopia\Database\Exception\Limit;
|
||||
use Utopia\Database\Exception\Authorization as ExceptionAuthorization;
|
||||
use Utopia\Database\Exception\Structure;
|
||||
use Utopia\Database\Query;
|
||||
use Utopia\Database\Validator\Authorization;
|
||||
|
||||
global $register;
|
||||
|
||||
class V11 extends Migration
|
||||
{
|
||||
protected Database $dbInternal;
|
||||
protected Database $dbExternal;
|
||||
protected Database $dbConsole;
|
||||
|
||||
protected array $oldCollections;
|
||||
protected array $newCollections;
|
||||
|
||||
public function __construct(PDO $db, Redis $cache = null, array $options = [])
|
||||
{
|
||||
parent::__construct($db, $cache, $options);
|
||||
$this->options = array_map(fn ($option) => $option === 'yes' ? true : false, $this->options);
|
||||
|
||||
if (!is_null($cache)) {
|
||||
$cacheAdapter = new Cache(new RedisCache($this->cache));
|
||||
$this->dbInternal = new Database(new MariaDB($this->db), $cacheAdapter); // namespace is set on execution
|
||||
$this->dbExternal = new Database(new MariaDB($this->db), $cacheAdapter); // namespace is set on execution
|
||||
$this->dbConsole = new Database(new MariaDB($this->db), $cacheAdapter);
|
||||
$this->dbConsole->setNamespace('project_console_internal');
|
||||
}
|
||||
|
||||
$this->newCollections = Config::getParam('collections', []);
|
||||
$this->oldCollections = Config::getParam('collectionsold', []);
|
||||
}
|
||||
|
||||
public function execute(): void
|
||||
{
|
||||
Authorization::disable();
|
||||
Runtime::enableCoroutine(SWOOLE_HOOK_ALL);
|
||||
|
||||
$oldProject = $this->project;
|
||||
|
||||
$this->dbInternal->setNamespace('project_' . $oldProject->getId() . '_internal');
|
||||
$this->dbExternal->setNamespace('project_' . $oldProject->getId() . '_external');
|
||||
|
||||
Console::info('');
|
||||
Console::info('------------------------------------');
|
||||
Console::info('Migrating project ' . $oldProject->getAttribute('name'));
|
||||
Console::info('------------------------------------');
|
||||
|
||||
/**
|
||||
* Create internal/external structure for projects and skip the console project.
|
||||
*/
|
||||
if ($oldProject->getId() !== 'console') {
|
||||
$project = $this->dbConsole->getDocument('projects', $oldProject->getId());
|
||||
|
||||
/**
|
||||
* Migrate Project Document.
|
||||
*/
|
||||
if ($project->isEmpty()) {
|
||||
$newProject = $this->fixDocument($oldProject);
|
||||
$newProject->setAttribute('version', '0.12.0');
|
||||
$project = $this->dbConsole->createDocument('projects', $newProject);
|
||||
Console::log('Created project document: ' . $oldProject->getAttribute('name') . ' (' . $oldProject->getId() . ')');
|
||||
}
|
||||
|
||||
/**
|
||||
* Create internal DB tables
|
||||
*/
|
||||
if (!$this->dbInternal->exists()) {
|
||||
$this->dbInternal->create();
|
||||
Console::log('Created internal tables for : ' . $project->getAttribute('name') . ' (' . $project->getId() . ')');
|
||||
}
|
||||
|
||||
/**
|
||||
* Create external DB tables
|
||||
*/
|
||||
if (!$this->dbExternal->exists()) {
|
||||
$this->dbExternal->create();
|
||||
Console::log('Created external tables for : ' . $project->getAttribute('name') . ' (' . $project->getId() . ')');
|
||||
}
|
||||
|
||||
/**
|
||||
* Create Audit tables
|
||||
*/
|
||||
if ($this->dbInternal->getCollection(Audit::COLLECTION)->isEmpty()) {
|
||||
$audit = new Audit($this->dbInternal);
|
||||
$audit->setup();
|
||||
Console::log('Created audit tables for : ' . $project->getAttribute('name') . ' (' . $project->getId() . ')');
|
||||
}
|
||||
|
||||
/**
|
||||
* Create Abuse tables
|
||||
*/
|
||||
if ($this->dbInternal->getCollection(TimeLimit::COLLECTION)->isEmpty()) {
|
||||
$adapter = new TimeLimit("", 0, 1, $this->dbInternal);
|
||||
$adapter->setup();
|
||||
Console::log('Created abuse tables for : ' . $project->getAttribute('name') . ' (' . $project->getId() . ')');
|
||||
}
|
||||
|
||||
/**
|
||||
* Create internal collections for Project
|
||||
*/
|
||||
foreach ($this->newCollections as $key => $collection) {
|
||||
if (!$this->dbInternal->getCollection($key)->isEmpty()) continue; // Skip if project collection already exists
|
||||
|
||||
$attributes = [];
|
||||
$indexes = [];
|
||||
|
||||
foreach ($collection['attributes'] as $attribute) {
|
||||
$attributes[] = new Document([
|
||||
'$id' => $attribute['$id'],
|
||||
'type' => $attribute['type'],
|
||||
'size' => $attribute['size'],
|
||||
'required' => $attribute['required'],
|
||||
'signed' => $attribute['signed'],
|
||||
'array' => $attribute['array'],
|
||||
'filters' => $attribute['filters'],
|
||||
]);
|
||||
}
|
||||
|
||||
foreach ($collection['indexes'] as $index) {
|
||||
$indexes[] = new Document([
|
||||
'$id' => $index['$id'],
|
||||
'type' => $index['type'],
|
||||
'attributes' => $index['attributes'],
|
||||
'lengths' => $index['lengths'],
|
||||
'orders' => $index['orders'],
|
||||
]);
|
||||
}
|
||||
|
||||
$this->dbInternal->createCollection($key, $attributes, $indexes);
|
||||
}
|
||||
if ($this->options['migrateCollections']) {
|
||||
$this->migrateExternalCollections();
|
||||
}
|
||||
} else {
|
||||
Console::log('Skipped console project migration.');
|
||||
}
|
||||
|
||||
$sum = $this->limit;
|
||||
$offset = 0;
|
||||
$total = 0;
|
||||
|
||||
/**
|
||||
* Migrate internal documents
|
||||
*/
|
||||
while ($sum >= $this->limit) {
|
||||
$all = $this->oldProjectDB->getCollection([
|
||||
'limit' => $this->limit,
|
||||
'offset' => $offset,
|
||||
'orderType' => 'DESC',
|
||||
'filters' => [
|
||||
'$collection!=' . OldDatabase::SYSTEM_COLLECTION_COLLECTIONS,
|
||||
'$collection!=' . OldDatabase::SYSTEM_COLLECTION_RULES,
|
||||
'$collection!=' . OldDatabase::SYSTEM_COLLECTION_TASKS,
|
||||
'$collection!=' . OldDatabase::SYSTEM_COLLECTION_PROJECTS,
|
||||
'$collection!=' . OldDatabase::SYSTEM_COLLECTION_CONNECTIONS,
|
||||
]
|
||||
]);
|
||||
|
||||
$sum = \count($all);
|
||||
|
||||
Console::log('Migrating Internal Documents: ' . $offset . ' / ' . $this->oldProjectDB->getSum());
|
||||
|
||||
foreach ($all as $document) {
|
||||
if (
|
||||
!array_key_exists($document->getCollection(), $this->oldCollections)
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$new = $this->fixDocument($document);
|
||||
|
||||
if (empty($new->getId())) {
|
||||
Console::warning('Skipped Document due to missing ID.');
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
if ($this->dbInternal->getDocument($new->getCollection(), $new->getId())->isEmpty()) {
|
||||
$this->dbInternal->createDocument($new->getCollection(), $new);
|
||||
}
|
||||
} catch (\Throwable $th) {
|
||||
Console::error('Failed to update document: ' . $th->getMessage());
|
||||
continue;
|
||||
|
||||
if ($document && $new->getId() !== $document->getId()) {
|
||||
throw new Exception('Duplication Error');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$offset += $this->limit;
|
||||
$total += $sum;
|
||||
}
|
||||
Console::log('Migrated ' . $total . ' Internal Documents.');
|
||||
}
|
||||
|
||||
/**
|
||||
* Migrate external collections for Project
|
||||
*
|
||||
* @return void
|
||||
* @throws Exception
|
||||
* @throws Throwable
|
||||
* @throws Limit
|
||||
* @throws ExceptionAuthorization
|
||||
* @throws Structure
|
||||
*/
|
||||
protected function migrateExternalCollections(): void
|
||||
{
|
||||
$sum = $this->limit;
|
||||
$offset = 0;
|
||||
|
||||
while ($sum >= $this->limit) {
|
||||
$databaseCollections = $this->oldProjectDB->getCollection([
|
||||
'limit' => $this->limit,
|
||||
'offset' => $offset,
|
||||
'orderType' => 'DESC',
|
||||
'filters' => [
|
||||
'$collection=' . OldDatabase::SYSTEM_COLLECTION_COLLECTIONS,
|
||||
]
|
||||
]);
|
||||
|
||||
$sum = \count($databaseCollections);
|
||||
Console::log('Migrating Collections: ' . $offset . ' / ' . $this->oldProjectDB->getSum());
|
||||
|
||||
foreach ($databaseCollections as $oldCollection) {
|
||||
$id = $oldCollection->getId();
|
||||
$permissions = $oldCollection->getPermissions();
|
||||
$name = $oldCollection->getAttribute('name');
|
||||
$newCollection = $this->dbExternal->getCollection($id);
|
||||
|
||||
if ($newCollection->isEmpty()) {
|
||||
$this->dbExternal->createCollection($id);
|
||||
/**
|
||||
* Migrate permissions
|
||||
*/
|
||||
$read = $this->migrateWildcardPermissions($permissions['read'] ?? []);
|
||||
$write = $this->migrateWildcardPermissions($permissions['write'] ?? []);
|
||||
|
||||
/**
|
||||
* Suffix collection name with a subsequent number to make it unique if possible.
|
||||
*/
|
||||
$suffix = 1;
|
||||
while ($this->dbInternal->findOne('collections', [
|
||||
new Query('name', Query::TYPE_EQUAL, [$name])
|
||||
])) {
|
||||
$name .= ' - ' . $suffix++;
|
||||
}
|
||||
|
||||
$this->dbInternal->createDocument('collections', new Document([
|
||||
'$id' => $id,
|
||||
'$read' => [],
|
||||
'$write' => [],
|
||||
'permission' => 'document',
|
||||
'dateCreated' => time(),
|
||||
'dateUpdated' => time(),
|
||||
'name' => $name,
|
||||
'search' => implode(' ', [$id, $name]),
|
||||
]));
|
||||
} else {
|
||||
Console::warning('Skipped Collection ' . $newCollection->getId() . ' from ' . $newCollection->getCollection());
|
||||
}
|
||||
|
||||
/**
|
||||
* Migrate collection rules to attributes
|
||||
*/
|
||||
$attributes = $this->getCollectionAttributes($oldCollection);
|
||||
|
||||
foreach ($attributes as $attribute) {
|
||||
try {
|
||||
$this->dbExternal->createAttribute(
|
||||
collection: $attribute['$collection'],
|
||||
id: $attribute['$id'],
|
||||
type: $attribute['type'],
|
||||
size: $attribute['size'],
|
||||
required: $attribute['required'],
|
||||
default: $attribute['default'],
|
||||
signed: $attribute['signed'],
|
||||
array: $attribute['array'],
|
||||
format: $attribute['format'] ?? null,
|
||||
formatOptions: $attribute['formatOptions'] ?? [],
|
||||
filters: $attribute['filters']
|
||||
);
|
||||
|
||||
$this->dbInternal->createDocument('attributes', new Document([
|
||||
'$id' => $attribute['$collection'] . '_' . $attribute['$id'],
|
||||
'key' => $attribute['$id'],
|
||||
'collectionId' => $attribute['$collection'],
|
||||
'type' => $attribute['type'],
|
||||
'status' => 'available',
|
||||
'size' => $attribute['size'],
|
||||
'required' => $attribute['required'],
|
||||
'signed' => $attribute['signed'],
|
||||
'default' => $attribute['default'],
|
||||
'array' => $attribute['array'],
|
||||
'format' => $attribute['format'] ?? null,
|
||||
'formatOptions' => $attribute['formatOptions'] ?? null,
|
||||
'filters' => $attribute['filters']
|
||||
]));
|
||||
|
||||
Console::log('Created "' . $attribute['$id'] . '" attribute in collection: ' . $name);
|
||||
} catch (\Throwable $th) {
|
||||
Console::log($th->getMessage() . ' - ("' . $attribute['$id'] . '" attribute in collection ' . $name . ')');
|
||||
}
|
||||
}
|
||||
if ($this->options['migrateDocuments']) {
|
||||
$this->migrateExternalDocuments(collection: $id);
|
||||
}
|
||||
}
|
||||
$offset += $this->limit;
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Migrate all external documents
|
||||
*
|
||||
* @return void
|
||||
* @throws Exception
|
||||
* @throws Throwable
|
||||
* @throws ExceptionAuthorization
|
||||
* @throws Structure
|
||||
*/
|
||||
protected function migrateExternalDocuments(string $collection): void
|
||||
{
|
||||
$sum = $this->limit;
|
||||
$offset = 0;
|
||||
|
||||
while ($sum >= $this->limit) {
|
||||
$allDocs = $this->oldProjectDB->getCollection([
|
||||
'limit' => $this->limit,
|
||||
'offset' => $offset,
|
||||
'orderType' => 'DESC',
|
||||
'filters' => [
|
||||
'$collection=' . $collection
|
||||
]
|
||||
]);
|
||||
|
||||
$sum = \count($allDocs);
|
||||
|
||||
Console::log('Migrating External Documents for Collection ' . $collection . ': ' . $offset . ' / ' . $this->oldProjectDB->getSum());
|
||||
|
||||
foreach ($allDocs as $document) {
|
||||
if (!$this->dbExternal->getDocument($collection, $document->getId())->isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
go(function ($document) {
|
||||
foreach ($document as $key => $attr) {
|
||||
/**
|
||||
* Convert nested Document to JSON strings.
|
||||
*/
|
||||
if ($document->getAttribute($key) instanceof OldDocument) {
|
||||
$document[$key] = json_encode($this->fixDocument($attr)->getArrayCopy());
|
||||
}
|
||||
/**
|
||||
* Convert numeric Attributes to float.
|
||||
*/
|
||||
if (is_numeric($attr)) {
|
||||
$document[$key] = floatval($attr);
|
||||
}
|
||||
if (\is_array($attr)) {
|
||||
foreach ($attr as $index => $child) {
|
||||
/**
|
||||
* Convert array of nested Document to array JSON strings.
|
||||
*/
|
||||
if ($document->getAttribute($key)[$index] instanceof OldDocument) {
|
||||
$document[$key][$index] = json_encode($this->fixDocument($child)->getArrayCopy());
|
||||
}
|
||||
/**
|
||||
* Convert array of numeric Attributes to array float.
|
||||
*/
|
||||
if (is_numeric($attr)) {
|
||||
$document[$key][$index] = floatval($child); // Convert any numeric to float
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}, $document);
|
||||
$document = new Document($document->getArrayCopy());
|
||||
$document = $this->migratePermissions($document);
|
||||
$this->dbExternal->createDocument($collection, $document);
|
||||
}
|
||||
$offset += $this->limit;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Migrates single docuemnt.
|
||||
*
|
||||
* @param OldDocument $oldDocument
|
||||
* @return Document
|
||||
* @throws Exception
|
||||
*/
|
||||
protected function fixDocument(OldDocument $oldDocument): Document
|
||||
{
|
||||
$document = new Document($oldDocument->getArrayCopy());
|
||||
$document = $this->migratePermissions($document);
|
||||
|
||||
/**
|
||||
* Check attributes and set their default values.
|
||||
*/
|
||||
if (array_key_exists($document->getCollection(), $this->oldCollections)) {
|
||||
foreach ($this->newCollections[$document->getCollection()]['attributes'] as $attr) {
|
||||
if (
|
||||
(!$attr['array'] ||
|
||||
($attr['array'] && array_key_exists('filter', $attr)
|
||||
&& in_array('json', $attr['filter'])))
|
||||
&& empty($document->getAttribute($attr['$id'], null))
|
||||
) {
|
||||
$document->setAttribute($attr['$id'], $attr['default'] ?? null);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
switch ($document->getAttribute('$collection')) {
|
||||
case OldDatabase::SYSTEM_COLLECTION_PLATFORMS:
|
||||
$projectId = $this->getProjectIdFromReadPermissions($document);
|
||||
|
||||
/**
|
||||
* Set Project ID
|
||||
*/
|
||||
if ($document->getAttribute('projectId') === null) {
|
||||
$document->setAttribute('projectId', $projectId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Set empty key and store if null
|
||||
*/
|
||||
if ($document->getAttribute('key') === null) {
|
||||
$document->setAttribute('key', '');
|
||||
}
|
||||
if ($document->getAttribute('store') === null) {
|
||||
$document->setAttribute('store', '');
|
||||
}
|
||||
|
||||
/**
|
||||
* Reset Permissions
|
||||
*/
|
||||
$document->setAttribute('$read', ['role:all']);
|
||||
$document->setAttribute('$write', ['role:all']);
|
||||
|
||||
break;
|
||||
case OldDatabase::SYSTEM_COLLECTION_DOMAINS:
|
||||
$projectId = $this->getProjectIdFromReadPermissions($document);
|
||||
|
||||
/**
|
||||
* Set Project ID
|
||||
*/
|
||||
if ($document->getAttribute('projectId') === null) {
|
||||
$document->setAttribute('projectId', $projectId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Set empty verification if null
|
||||
*/
|
||||
if ($document->getAttribute('verification') === null) {
|
||||
$document->setAttribute('verification', false);
|
||||
}
|
||||
|
||||
/**
|
||||
* Reset Permissions
|
||||
*/
|
||||
$document->setAttribute('$read', ['role:all']);
|
||||
$document->setAttribute('$write', ['role:all']);
|
||||
|
||||
break;
|
||||
case OldDatabase::SYSTEM_COLLECTION_KEYS:
|
||||
case OldDatabase::SYSTEM_COLLECTION_WEBHOOKS:
|
||||
$projectId = $this->getProjectIdFromReadPermissions($document);
|
||||
|
||||
/**
|
||||
* Set Project ID
|
||||
*/
|
||||
if ($document->getAttribute('projectId') === null) {
|
||||
$document->setAttribute('projectId', $projectId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Reset Permissions
|
||||
*/
|
||||
$document->setAttribute('$read', ['role:all']);
|
||||
$document->setAttribute('$write', ['role:all']);
|
||||
|
||||
break;
|
||||
case OldDatabase::SYSTEM_COLLECTION_USERS:
|
||||
/**
|
||||
* Set deleted attribute to false
|
||||
*/
|
||||
if ($document->getAttribute('deleted') === null) {
|
||||
$document->setAttribute('deleted', false);
|
||||
}
|
||||
/**
|
||||
* Remove deprecated user status 0 and replace with boolean.
|
||||
*/
|
||||
if ($document->getAttribute('status') === 2) {
|
||||
$document->setAttribute('status', false);
|
||||
} else {
|
||||
$document->setAttribute('status', true);
|
||||
}
|
||||
|
||||
/**
|
||||
* Set default values for arrays if not set.
|
||||
*/
|
||||
if (empty($document->getAttribute('prefs', []))) {
|
||||
$document->setAttribute('prefs', []);
|
||||
}
|
||||
if (empty($document->getAttribute('sessions', []))) {
|
||||
$document->setAttribute('sessions', []);
|
||||
}
|
||||
if (empty($document->getAttribute('tokens', []))) {
|
||||
$document->setAttribute('tokens', []);
|
||||
}
|
||||
if (empty($document->getAttribute('memberships', []))) {
|
||||
$document->setAttribute('memberships', []);
|
||||
}
|
||||
|
||||
/**
|
||||
* Replace user:{self} with user:USER_ID
|
||||
*/
|
||||
$write = $document->getWrite();
|
||||
$document->setAttribute('$write', str_replace('user:{self}', "user:{$document->getId()}", $write));
|
||||
|
||||
break;
|
||||
case OldDatabase::SYSTEM_COLLECTION_FILES:
|
||||
/**
|
||||
* Migrating breakind changes on Files.
|
||||
*/
|
||||
if (!empty($document->getAttribute('fileOpenSSLVersion', null))) {
|
||||
$document
|
||||
->setAttribute('openSSLVersion', $document->getAttribute('fileOpenSSLVersion'))
|
||||
->removeAttribute('fileOpenSSLVersion');
|
||||
}
|
||||
if (!empty($document->getAttribute('fileOpenSSLCipher', null))) {
|
||||
$document
|
||||
->setAttribute('openSSLCipher', $document->getAttribute('fileOpenSSLCipher'))
|
||||
->removeAttribute('fileOpenSSLCipher');
|
||||
}
|
||||
if (!empty($document->getAttribute('fileOpenSSLTag', null))) {
|
||||
$document
|
||||
->setAttribute('openSSLTag', $document->getAttribute('fileOpenSSLTag'))
|
||||
->removeAttribute('fileOpenSSLTag');
|
||||
}
|
||||
if (!empty($document->getAttribute('fileOpenSSLIV', null))) {
|
||||
$document
|
||||
->setAttribute('openSSLIV', $document->getAttribute('fileOpenSSLIV'))
|
||||
->removeAttribute('fileOpenSSLIV');
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove deprecated attributes.
|
||||
*/
|
||||
$document->removeAttribute('folderId');
|
||||
$document->removeAttribute('token');
|
||||
break;
|
||||
}
|
||||
|
||||
return $document;
|
||||
}
|
||||
|
||||
/**
|
||||
* Migrates $permissions to independent $read and $write.
|
||||
* @param Document $document
|
||||
* @return Document
|
||||
*/
|
||||
protected function migratePermissions(Document $document): Document
|
||||
{
|
||||
if ($document->isSet('$permissions')) {
|
||||
$permissions = $document->getAttribute('$permissions', []);
|
||||
$read = $this->migrateWildcardPermissions($permissions['read'] ?? []);
|
||||
$write = $this->migrateWildcardPermissions($permissions['write'] ?? []);
|
||||
$document->setAttribute('$read', $read);
|
||||
$document->setAttribute('$write', $write);
|
||||
$document->removeAttribute('$permissions');
|
||||
}
|
||||
|
||||
return $document;
|
||||
}
|
||||
|
||||
/**
|
||||
* Takes a permissions array and replaces wildcard * with role:all.
|
||||
* @param array $permissions
|
||||
* @return array
|
||||
*/
|
||||
protected function migrateWildcardPermissions(array $permissions): array
|
||||
{
|
||||
return array_map(function ($permission) {
|
||||
if ($permission === '*') return 'role:all';
|
||||
return $permission;
|
||||
}, $permissions);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get new collection attributes from old collection rules.
|
||||
* @param OldDocument $collection
|
||||
* @return array
|
||||
*/
|
||||
protected function getCollectionAttributes(OldDocument $collection): array
|
||||
{
|
||||
$attributes = [];
|
||||
foreach ($collection->getAttribute('rules', []) as $key => $value) {
|
||||
$collectionId = $collection->getId();
|
||||
$id = $value['key'];
|
||||
$array = $value['array'] ?? false;
|
||||
$required = $value['required'] ?? false;
|
||||
$default = $value['default'] ?? null;
|
||||
$default = match ($value['type']) {
|
||||
OldDatabase::SYSTEM_VAR_TYPE_NUMERIC => floatval($default),
|
||||
default => $default
|
||||
};
|
||||
$type = match ($value['type']) {
|
||||
OldDatabase::SYSTEM_VAR_TYPE_TEXT => Database::VAR_STRING,
|
||||
OldDatabase::SYSTEM_VAR_TYPE_EMAIL => Database::VAR_STRING,
|
||||
OldDatabase::SYSTEM_VAR_TYPE_DOCUMENT => Database::VAR_STRING,
|
||||
OldDatabase::SYSTEM_VAR_TYPE_IP => Database::VAR_STRING,
|
||||
OldDatabase::SYSTEM_VAR_TYPE_URL => Database::VAR_STRING,
|
||||
OldDatabase::SYSTEM_VAR_TYPE_WILDCARD => Database::VAR_STRING,
|
||||
OldDatabase::SYSTEM_VAR_TYPE_NUMERIC => Database::VAR_FLOAT,
|
||||
OldDatabase::SYSTEM_VAR_TYPE_BOOLEAN => Database::VAR_BOOLEAN,
|
||||
default => Database::VAR_STRING
|
||||
};
|
||||
|
||||
$size = $type === Database::VAR_STRING ? 65_535 : 0; // Max size of text in MariaDB
|
||||
|
||||
$attributes[$key] = [
|
||||
'$collection' => $collectionId,
|
||||
'$id' => $id,
|
||||
'type' => $type,
|
||||
'size' => $size,
|
||||
'required' => $required,
|
||||
'default' => $default,
|
||||
'array' => $array,
|
||||
'signed' => true,
|
||||
'filters' => []
|
||||
];
|
||||
|
||||
if ($type === Database::VAR_FLOAT) {
|
||||
$attributes[$key]['format'] = APP_DATABASE_ATTRIBUTE_FLOAT_RANGE;
|
||||
$attributes[$key]['formatOptions'] = [];
|
||||
$attributes[$key]['formatOptions']['min'] = -PHP_FLOAT_MAX;
|
||||
$attributes[$key]['formatOptions']['max'] = PHP_FLOAT_MAX;
|
||||
}
|
||||
}
|
||||
|
||||
return $attributes;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param Document $document
|
||||
* @return string|null
|
||||
* @throws Exception
|
||||
*/
|
||||
protected function getProjectIdFromReadPermissions(Document $document): string|null
|
||||
{
|
||||
$readPermissions = $document->getRead();
|
||||
$teamId = str_replace('team:', '', reset($readPermissions));
|
||||
return $this->oldConsoleDB->getCollectionFirst([
|
||||
'filters' => [
|
||||
'$collection=' . OldDatabase::SYSTEM_COLLECTION_PROJECTS,
|
||||
'teamId=' . $teamId
|
||||
]
|
||||
])->getId();
|
||||
}
|
||||
}
|
Loading…
Reference in a new issue