CodeMirror/appwrite
1
0
Fork 0
mirror of synced 2024-06-26 10:10:57 +12:00
Code Issues Projects Releases Wiki Activity

Merge pull request #2474 from appwrite/feat-database-indexing-migration

Browse source 
feat(tasks): migration for 0.12.x
This commit is contained in:
Torsten Dittmann 2021-12-21 17:34:41 +01:00 committed by GitHub
parent c91db74b23 d482fb86ea
commit 166ebf7225
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 875 additions and 46 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
app/config/collections.old.php
View file

@ -1617,8 +1617,8 @@ foreach ($providers as $index => $provider) {
foreach ($auth as $index => $method) {
$collections[Database::SYSTEM_COLLECTION_PROJECTS]['rules'][] = [
'$collection' => Database::SYSTEM_COLLECTION_RULES,
'label' => $method['name'] || '',
'key' => $method['key'] || '',
'label' => $method['name'] ?? '',
'key' => $method['key'] ?? '',
'type' => Database::SYSTEM_VAR_TYPE_BOOLEAN,
'default' => true,
'required' => false,

20
app/config/collections.php
View file

@ -487,7 +487,7 @@ $collections = [
'size' => 16384,
'signed' => true,
'required' => false,
'default' => null,
'default' => [],
'array' => false,
'filters' => ['json'],
],
@ -498,7 +498,7 @@ $collections = [
'size' => 16384,
'signed' => true,
'required' => false,
'default' => null,
'default' => [],
'array' => false,
'filters' => ['json'],
],
@ -509,7 +509,7 @@ $collections = [
'size' => 16384,
'signed' => true,
'required' => false,
'default' => null,
'default' => [],
'array' => false,
'filters' => ['json', 'encrypt'],
],
@ -884,7 +884,7 @@ $collections = [
'format' => '',
'size' => Database::LENGTH_KEY,
'signed' => true,
'required' => true,
'required' => false,
'default' => null,
'array' => false,
'filters' => [],
@ -895,7 +895,7 @@ $collections = [
'format' => '',
'size' => Database::LENGTH_KEY, // TODO will the length suffice after encryption?
'signed' => true,
'required' => true,
'required' => false,
'default' => null,
'array' => false,
'filters' => ['encrypt'],
@ -1001,7 +1001,7 @@ $collections = [
'size' => 16384,
'signed' => true,
'required' => false,
'default' => null,
'default' => [],
'array' => false,
'filters' => ['json'],
],
@ -1045,7 +1045,7 @@ $collections = [
'size' => 16384,
'signed' => true,
'required' => false,
'default' => null,
'default' => [],
'array' => true,
'filters' => ['json'],
],
@ -1056,7 +1056,7 @@ $collections = [
'size' => 16384,
'signed' => true,
'required' => false,
'default' => null,
'default' => [],
'array' => true,
'filters' => ['json'],
],
@ -1067,7 +1067,7 @@ $collections = [
'size' => 16384,
'signed' => true,
'required' => false,
'default' => null,
'default' => [],
'array' => true,
'filters' => ['json'],
],
@ -1816,7 +1816,7 @@ $collections = [
'size' => 8192,
'signed' => true,
'required' => false,
'default' => null,
'default' => [],
'array' => false,
'filters' => ['json', 'encrypt'],
],

42
app/init.php
View file

@ -21,6 +21,7 @@ use Appwrite\Extend\PDO;
use Ahc\Jwt\JWT;
use Ahc\Jwt\JWTException;
use Appwrite\Auth\Auth;
use Appwrite\Database\Database as DatabaseOld;
use Appwrite\Event\Event;
use Appwrite\Network\Validator\Email;
use Appwrite\Network\Validator\IP;
@ -153,6 +154,43 @@ if(!empty($user) || !empty($pass)) {
Resque::setBackend(App::getEnv('_APP_REDIS_HOST', '').':'.App::getEnv('_APP_REDIS_PORT', ''));
}
/**
* Old DB Filters
*/
DatabaseOld::addFilter('json',
function($value) {
if(!is_array($value)) {
return $value;
}
return json_encode($value);
},
function($value) {
return json_decode($value, true);
}
);
DatabaseOld::addFilter('encrypt',
function($value) {
$key = App::getEnv('_APP_OPENSSL_KEY_V1');
$iv = OpenSSL::randomPseudoBytes(OpenSSL::cipherIVLength(OpenSSL::CIPHER_AES_128_GCM));
$tag = null;
return json_encode([
'data' => OpenSSL::encrypt($value, OpenSSL::CIPHER_AES_128_GCM, $key, 0, $iv, $tag),
'method' => OpenSSL::CIPHER_AES_128_GCM,
'iv' => bin2hex($iv),
'tag' => bin2hex($tag),
'version' => '1',
]);
},
function($value) {
$value = json_decode($value, true);
$key = App::getEnv('_APP_OPENSSL_KEY_V'.$value['version']);
return OpenSSL::decrypt($value['data'], $value['method'], $key, 0, hex2bin($value['iv']), hex2bin($value['tag']));
}
);
/**
* New DB Filters
*/
@ -176,7 +214,7 @@ Database::addFilter('enum',
return $value;
},
function($value, Document $attribute) {
$formatOptions = json_decode($attribute->getAttribute('formatOptions', []), true);
$formatOptions = json_decode($attribute->getAttribute('formatOptions', '[]'), true);
if (isset($formatOptions['elements'])) {
$attribute->setAttribute('elements', $formatOptions['elements']);
}
@ -195,7 +233,7 @@ Database::addFilter('range',
return $value;
},
function($value, Document $attribute) {
$formatOptions = json_decode($attribute->getAttribute('formatOptions', []), true);
$formatOptions = json_decode($attribute->getAttribute('formatOptions', '[]'), true);
if (isset($formatOptions['min']) || isset($formatOptions['max'])) {
$attribute
->setAttribute('min', $formatOptions['min'])

77
app/tasks/migrate.php
View file

@ -11,55 +11,106 @@ use Appwrite\Database\Adapter\Redis as RedisAdapter;
use Appwrite\Migration\Migration;
use Utopia\Validator\Text;
Config::load('collections.old', __DIR__.'/../config/collections.old.php');
Config::load('collections.old', __DIR__ . '/../config/collections.old.php');
$cli
->task('migrate')
->param('version', APP_VERSION_STABLE, new Text(8), 'Version to migrate to.', true)
->action(function ($version) use ($register) {
Authorization::disable();
if (!array_key_exists($version, Migration::$versions)) {
Console::error("Version {$version} not found.");
Console::exit(1);
return;
}
$options = [];
if (str_starts_with($version, '0.12.')) {
Console::error('--------------------');
Console::error('WARNING');
Console::error('--------------------');
Console::warning('Migrating to Version 0.12.x introduces a major breaking change within the Database Service!');
Console::warning('Before migrating, please read about the breaking changes here:');
Console::info('https://appwrite.io/guide-to-db-migration');
$confirm = Console::confirm("If you want to proceed, type 'yes':");
if ($confirm != 'yes') {
Console::exit(1);
return;
}
Console::log('');
Console::log('Collections');
Console::log('--------------------');
Console::warning('Be aware that following actions will happen during the migration:');
Console::warning('- Nested Document rules will be migrated to String attributes');
Console::warning('- Numeric rules will be migrated to float attributes');
Console::warning('- Wildcard and Markdown rules will be converted to string attributes');
Console::info("Do you want to migrate your Database Collections?");
$options['migrateCollections'] = Console::confirm("Type 'yes' or 'no':");
if ($options['migrateCollections'] === 'yes') {
Console::log('');
Console::log('Documents');
Console::log('------------------');
Console::warning('Be aware that following actions will happen during the migration:');
Console::warning('- Nested Documents will be stored as JSON values');
Console::warning('- All Numeric values will be converted to float');
Console::warning('- All Wildcard and Markdown values will be converted to string');
Console::info("Do you want to migrate your Database Documents?");
$options['migrateDocuments'] = Console::confirm("Type 'yes' or 'no':");
} else {
$options['migrateDocuments'] = 'no';
}
if (
!in_array($options['migrateDocuments'], ['yes', 'no'])
|| !in_array($options['migrateCollections'], ['yes', 'no'])
) {
Console::error("You must reply with 'yes' or 'no'!");
Console::exit(1);
return;
}
}
Config::load('collectionsold', __DIR__ . '/../config/collections.old.php');
Console::success('Starting Data Migration to version ' . $version);
Console::success('Starting Data Migration to version '.$version);
$db = $register->get('db', true);
$cache = $register->get('cache', true);
$cache->flushAll();
$consoleDB = new Database();
$consoleDB
->setAdapter(new RedisAdapter(new MySQLAdapter($db, $cache), $cache))
->setNamespace('app_console') // Main DB
->setMocks(Config::getParam('collections.old', []));
->setMocks(Config::getParam('collectionsold', []));
$projectDB = new Database();
$projectDB
->setAdapter(new RedisAdapter(new MySQLAdapter($db, $cache), $cache))
->setMocks(Config::getParam('collections.old', []));
->setMocks(Config::getParam('collectionsold', []));
$console = $consoleDB->getDocument('console');
Authorization::disable();
$limit = 30;
$sum = 30;
$offset = 0;
$projects = [$console];
$count = 0;
$class = 'Appwrite\\Migration\\Version\\'.Migration::$versions[$version];
$migration = new $class($register->get('db'));
$class = 'Appwrite\\Migration\\Version\\' . Migration::$versions[$version];
$migration = new $class($register->get('db'), $register->get('cache'), $options);
while ($sum > 0) {
foreach ($projects as $project) {
try {
$migration
->setProject($project, $projectDB)
->setProject($project, $projectDB, $consoleDB)
->execute();
} catch (\Throwable $th) {
throw $th;
Console::error('Failed to update project ("'.$project->getId().'") version with error: '.$th->getMessage());
Console::error('Failed to update project ("' . $project->getId() . '") version with error: ' . $th->getMessage());
}
}
@ -67,7 +118,7 @@ $cli
'limit' => $limit,
'offset' => $offset,
'filters' => [
'$collection='.Database::SYSTEM_COLLECTION_PROJECTS,
'$collection=' . Database::SYSTEM_COLLECTION_PROJECTS,
],
]);
@ -76,9 +127,11 @@ $cli
$count = $count + $sum;
if ($sum > 0) {
Console::log('Fetched '.$count.'/'.$consoleDB->getSum().' projects...');
Console::log('Fetched ' . $count . '/' . $consoleDB->getSum() . ' projects...');
}
}
$cache->flushAll();
Swoole\Event::wait(); // Wait for Coroutines to finish
Console::success('Data Migration Completed');
});

22
src/Appwrite/Database/Document.php
View file

@ -91,6 +91,26 @@ class Document extends ArrayObject
return $temp;
}
/**
* Get Document Attributes
*
* @return array
*/
public function getAttributes(): array
{
$attributes = [];
foreach ($this as $attribute => $value) {
if(array_key_exists($attribute, ['$id' => true, '$permissions' => true, '$collection' => true, '$execute' => []])) {
continue;
}
$attributes[$attribute] = $value;
}
return $attributes;
}
/**
* Set Attribute.
*
@ -215,7 +235,7 @@ class Document extends ArrayObject
*
* @return array
*/
public function getArrayCopy(array $whitelist = [], array $blacklist = [])
public function getArrayCopy(array $whitelist = [], array $blacklist = []): array
{
$array = parent::getArrayCopy();

73
src/Appwrite/Migration/Migration.php
View file

@ -2,34 +2,50 @@
namespace Appwrite\Migration;
use Appwrite\Database\Document;
use Appwrite\Database\Database;
use Appwrite\Database\Document as OldDocument;
use Appwrite\Database\Database as OldDatabase;
use PDO;
use Redis;
use Swoole\Runtime;
use Utopia\CLI\Console;
use Utopia\Exception;
abstract class Migration
{
/**
* @var array
*/
protected array $options;
/**
* @var PDO
*/
protected $db;
protected PDO $db;
/**
* @var Redis
*/
protected Redis $cache;
/**
* @var int
*/
protected $limit = 50;
protected int $limit = 500;
/**
* @var Document
* @var OldDocument
*/
protected $project;
protected OldDocument $project;
/**
* @var Database
* @var OldDatabase
*/
protected $projectDB;
protected OldDatabase $oldProjectDB;
/**
* @var OldDatabase
*/
protected OldDatabase $oldConsoleDB;
/**
* @var array
@ -49,32 +65,44 @@ abstract class Migration
'0.10.3' => 'V09',
'0.10.4' => 'V09',
'0.11.0' => 'V10',
'0.12.0' => 'V10',
'0.12.0' => 'V11',
];
/**
* Migration constructor.
*
* @param PDO $pdo
* @param PDO $db
* @param Redis|null $cache
* @param array $options
* @return void
*/
public function __construct(PDO $db)
public function __construct(PDO $db, Redis $cache = null, array $options = [])
{
$this->options = $options;
$this->db = $db;
if (!is_null($cache)) {
$this->cache = $cache;
}
}
/**
* Set project for migration.
*
* @param Document $project
* @param Database $projectDB
* @param OldDocument $project
* @param OldDatabase $projectDB
* @param OldDatabase $oldConsoleDB
*
* @return Migration
* @return self
*/
public function setProject(Document $project, Database $projectDB): Migration
public function setProject(OldDocument $project, OldDatabase $projectDB, OldDatabase $oldConsoleDB): self
{
$this->project = $project;
$this->projectDB = $projectDB;
$this->projectDB->setNamespace('app_' . $project->getId());
$this->oldProjectDB = $projectDB;
$this->oldProjectDB->setNamespace('app_' . $project->getId());
$this->oldConsoleDB = $oldConsoleDB;
return $this;
}
@ -117,7 +145,7 @@ abstract class Migration
}
try {
$new = $this->projectDB->overwriteDocument($document->getArrayCopy());
$new = $this->projectDB->overwriteDocument($new->getArrayCopy());
} catch (\Throwable $th) {
Console::error('Failed to update document: ' . $th->getMessage());
return;
@ -134,7 +162,14 @@ abstract class Migration
}
}
public function check_diff_multi($array1, $array2)
/**
* Checks 2 arrays for differences.
*
* @param array $array1
* @param array $array2
* @return array
*/
public function check_diff_multi(array $array1, array $array2): array
{
$result = array();

683
src/Appwrite/Migration/Version/V11.php Normal file
View file

@ -0,0 +1,683 @@
<?php
namespace Appwrite\Migration\Version;
use Appwrite\Database\Database as OldDatabase;
use Appwrite\Database\Document as OldDocument;
use Appwrite\Migration\Migration;
use Exception;
use PDO;
use Redis;
use Swoole\Runtime;
use Throwable;
use Utopia\Abuse\Adapters\TimeLimit;
use Utopia\Audit\Audit;
use Utopia\Cache\Cache;
use Utopia\CLI\Console;
use Utopia\Cache\Adapter\Redis as RedisCache;
use Utopia\Config\Config;
use Utopia\Database\Adapter\MariaDB;
use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\Database\Exception\Limit;
use Utopia\Database\Exception\Authorization as ExceptionAuthorization;
use Utopia\Database\Exception\Structure;
use Utopia\Database\Query;
use Utopia\Database\Validator\Authorization;
global $register;
class V11 extends Migration
{
protected Database $dbInternal;
protected Database $dbExternal;
protected Database $dbConsole;
protected array $oldCollections;
protected array $newCollections;
public function __construct(PDO $db, Redis $cache = null, array $options = [])
{
parent::__construct($db, $cache, $options);
$this->options = array_map(fn ($option) => $option === 'yes' ? true : false, $this->options);
if (!is_null($cache)) {
$cacheAdapter = new Cache(new RedisCache($this->cache));
$this->dbInternal = new Database(new MariaDB($this->db), $cacheAdapter); // namespace is set on execution
$this->dbExternal = new Database(new MariaDB($this->db), $cacheAdapter); // namespace is set on execution
$this->dbConsole = new Database(new MariaDB($this->db), $cacheAdapter);
$this->dbConsole->setNamespace('project_console_internal');
}
$this->newCollections = Config::getParam('collections', []);
$this->oldCollections = Config::getParam('collectionsold', []);
}
public function execute(): void
{
Authorization::disable();
Runtime::enableCoroutine(SWOOLE_HOOK_ALL);
$oldProject = $this->project;
$this->dbInternal->setNamespace('project_' . $oldProject->getId() . '_internal');
$this->dbExternal->setNamespace('project_' . $oldProject->getId() . '_external');
Console::info('');
Console::info('------------------------------------');
Console::info('Migrating project ' . $oldProject->getAttribute('name'));
Console::info('------------------------------------');
/**
* Create internal/external structure for projects and skip the console project.
*/
if ($oldProject->getId() !== 'console') {
$project = $this->dbConsole->getDocument('projects', $oldProject->getId());
/**
* Migrate Project Document.
*/
if ($project->isEmpty()) {
$newProject = $this->fixDocument($oldProject);
$newProject->setAttribute('version', '0.12.0');
$project = $this->dbConsole->createDocument('projects', $newProject);
Console::log('Created project document: ' . $oldProject->getAttribute('name') . ' (' . $oldProject->getId() . ')');
}
/**
* Create internal DB tables
*/
if (!$this->dbInternal->exists()) {
$this->dbInternal->create();
Console::log('Created internal tables for : ' . $project->getAttribute('name') . ' (' . $project->getId() . ')');
}
/**
* Create external DB tables
*/
if (!$this->dbExternal->exists()) {
$this->dbExternal->create();
Console::log('Created external tables for : ' . $project->getAttribute('name') . ' (' . $project->getId() . ')');
}
/**
* Create Audit tables
*/
if ($this->dbInternal->getCollection(Audit::COLLECTION)->isEmpty()) {
$audit = new Audit($this->dbInternal);
$audit->setup();
Console::log('Created audit tables for : ' . $project->getAttribute('name') . ' (' . $project->getId() . ')');
}
/**
* Create Abuse tables
*/
if ($this->dbInternal->getCollection(TimeLimit::COLLECTION)->isEmpty()) {
$adapter = new TimeLimit("", 0, 1, $this->dbInternal);
$adapter->setup();
Console::log('Created abuse tables for : ' . $project->getAttribute('name') . ' (' . $project->getId() . ')');
}
/**
* Create internal collections for Project
*/
foreach ($this->newCollections as $key => $collection) {
if (!$this->dbInternal->getCollection($key)->isEmpty()) continue; // Skip if project collection already exists
$attributes = [];
$indexes = [];
foreach ($collection['attributes'] as $attribute) {
$attributes[] = new Document([
'$id' => $attribute['$id'],
'type' => $attribute['type'],
'size' => $attribute['size'],
'required' => $attribute['required'],
'signed' => $attribute['signed'],
'array' => $attribute['array'],
'filters' => $attribute['filters'],
]);
}
foreach ($collection['indexes'] as $index) {
$indexes[] = new Document([
'$id' => $index['$id'],
'type' => $index['type'],
'attributes' => $index['attributes'],
'lengths' => $index['lengths'],
'orders' => $index['orders'],
]);
}
$this->dbInternal->createCollection($key, $attributes, $indexes);
}
if ($this->options['migrateCollections']) {
$this->migrateExternalCollections();
}
} else {
Console::log('Skipped console project migration.');
}
$sum = $this->limit;
$offset = 0;
$total = 0;
/**
* Migrate internal documents
*/
while ($sum >= $this->limit) {
$all = $this->oldProjectDB->getCollection([
'limit' => $this->limit,
'offset' => $offset,
'orderType' => 'DESC',
'filters' => [
'$collection!=' . OldDatabase::SYSTEM_COLLECTION_COLLECTIONS,
'$collection!=' . OldDatabase::SYSTEM_COLLECTION_RULES,
'$collection!=' . OldDatabase::SYSTEM_COLLECTION_TASKS,
'$collection!=' . OldDatabase::SYSTEM_COLLECTION_PROJECTS,
'$collection!=' . OldDatabase::SYSTEM_COLLECTION_CONNECTIONS,
]
]);
$sum = \count($all);
Console::log('Migrating Internal Documents: ' . $offset . ' / ' . $this->oldProjectDB->getSum());
foreach ($all as $document) {
if (
!array_key_exists($document->getCollection(), $this->oldCollections)
) {
continue;
}
$new = $this->fixDocument($document);
if (empty($new->getId())) {
Console::warning('Skipped Document due to missing ID.');
continue;
}
try {
if ($this->dbInternal->getDocument($new->getCollection(), $new->getId())->isEmpty()) {
$this->dbInternal->createDocument($new->getCollection(), $new);
}
} catch (\Throwable $th) {
Console::error('Failed to update document: ' . $th->getMessage());
continue;
if ($document && $new->getId() !== $document->getId()) {
throw new Exception('Duplication Error');
}
}
}
$offset += $this->limit;
$total += $sum;
}
Console::log('Migrated ' . $total . ' Internal Documents.');
}
/**
* Migrate external collections for Project
*
* @return void
* @throws Exception
* @throws Throwable
* @throws Limit
* @throws ExceptionAuthorization
* @throws Structure
*/
protected function migrateExternalCollections(): void
{
$sum = $this->limit;
$offset = 0;
while ($sum >= $this->limit) {
$databaseCollections = $this->oldProjectDB->getCollection([
'limit' => $this->limit,
'offset' => $offset,
'orderType' => 'DESC',
'filters' => [
'$collection=' . OldDatabase::SYSTEM_COLLECTION_COLLECTIONS,
]
]);
$sum = \count($databaseCollections);
Console::log('Migrating Collections: ' . $offset . ' / ' . $this->oldProjectDB->getSum());
foreach ($databaseCollections as $oldCollection) {
$id = $oldCollection->getId();
$permissions = $oldCollection->getPermissions();
$name = $oldCollection->getAttribute('name');
$newCollection = $this->dbExternal->getCollection($id);
if ($newCollection->isEmpty()) {
$this->dbExternal->createCollection($id);
/**
* Migrate permissions
*/
$read = $this->migrateWildcardPermissions($permissions['read'] ?? []);
$write = $this->migrateWildcardPermissions($permissions['write'] ?? []);
/**
* Suffix collection name with a subsequent number to make it unique if possible.
*/
$suffix = 1;
while ($this->dbInternal->findOne('collections', [
new Query('name', Query::TYPE_EQUAL, [$name])
])) {
$name .= ' - ' . $suffix++;
}
$this->dbInternal->createDocument('collections', new Document([
'$id' => $id,
'$read' => [],
'$write' => [],
'permission' => 'document',
'dateCreated' => time(),
'dateUpdated' => time(),
'name' => $name,
'search' => implode(' ', [$id, $name]),
]));
} else {
Console::warning('Skipped Collection ' . $newCollection->getId() . ' from ' . $newCollection->getCollection());
}
/**
* Migrate collection rules to attributes
*/
$attributes = $this->getCollectionAttributes($oldCollection);
foreach ($attributes as $attribute) {
try {
$this->dbExternal->createAttribute(
collection: $attribute['$collection'],
id: $attribute['$id'],
type: $attribute['type'],
size: $attribute['size'],
required: $attribute['required'],
default: $attribute['default'],
signed: $attribute['signed'],
array: $attribute['array'],
format: $attribute['format'] ?? null,
formatOptions: $attribute['formatOptions'] ?? [],
filters: $attribute['filters']
);
$this->dbInternal->createDocument('attributes', new Document([
'$id' => $attribute['$collection'] . '_' . $attribute['$id'],
'key' => $attribute['$id'],
'collectionId' => $attribute['$collection'],
'type' => $attribute['type'],
'status' => 'available',
'size' => $attribute['size'],
'required' => $attribute['required'],
'signed' => $attribute['signed'],
'default' => $attribute['default'],
'array' => $attribute['array'],
'format' => $attribute['format'] ?? null,
'formatOptions' => $attribute['formatOptions'] ?? null,
'filters' => $attribute['filters']
]));
Console::log('Created "' . $attribute['$id'] . '" attribute in collection: ' . $name);
} catch (\Throwable $th) {
Console::log($th->getMessage() . ' - ("' . $attribute['$id'] . '" attribute in collection ' . $name . ')');
}
}
if ($this->options['migrateDocuments']) {
$this->migrateExternalDocuments(collection: $id);
}
}
$offset += $this->limit;
}
}
/**
* Migrate all external documents
*
* @return void
* @throws Exception
* @throws Throwable
* @throws ExceptionAuthorization
* @throws Structure
*/
protected function migrateExternalDocuments(string $collection): void
{
$sum = $this->limit;
$offset = 0;
while ($sum >= $this->limit) {
$allDocs = $this->oldProjectDB->getCollection([
'limit' => $this->limit,
'offset' => $offset,
'orderType' => 'DESC',
'filters' => [
'$collection=' . $collection
]
]);
$sum = \count($allDocs);
Console::log('Migrating External Documents for Collection ' . $collection . ': ' . $offset . ' / ' . $this->oldProjectDB->getSum());
foreach ($allDocs as $document) {
if (!$this->dbExternal->getDocument($collection, $document->getId())->isEmpty()) {
continue;
}
go(function ($document) {
foreach ($document as $key => $attr) {
/**
* Convert nested Document to JSON strings.
*/
if ($document->getAttribute($key) instanceof OldDocument) {
$document[$key] = json_encode($this->fixDocument($attr)->getArrayCopy());
}
/**
* Convert numeric Attributes to float.
*/
if (is_numeric($attr)) {
$document[$key] = floatval($attr);
}
if (\is_array($attr)) {
foreach ($attr as $index => $child) {
/**
* Convert array of nested Document to array JSON strings.
*/
if ($document->getAttribute($key)[$index] instanceof OldDocument) {
$document[$key][$index] = json_encode($this->fixDocument($child)->getArrayCopy());
}
/**
* Convert array of numeric Attributes to array float.
*/
if (is_numeric($attr)) {
$document[$key][$index] = floatval($child); // Convert any numeric to float
}
}
}
}
}, $document);
$document = new Document($document->getArrayCopy());
$document = $this->migratePermissions($document);
$this->dbExternal->createDocument($collection, $document);
}
$offset += $this->limit;
}
}
/**
* Migrates single docuemnt.
*
* @param OldDocument $oldDocument
* @return Document
* @throws Exception
*/
protected function fixDocument(OldDocument $oldDocument): Document
{
$document = new Document($oldDocument->getArrayCopy());
$document = $this->migratePermissions($document);
/**
* Check attributes and set their default values.
*/
if (array_key_exists($document->getCollection(), $this->oldCollections)) {
foreach ($this->newCollections[$document->getCollection()]['attributes'] as $attr) {
if (
(!$attr['array'] ||
($attr['array'] && array_key_exists('filter', $attr)
&& in_array('json', $attr['filter'])))
&& empty($document->getAttribute($attr['$id'], null))
) {
$document->setAttribute($attr['$id'], $attr['default'] ?? null);
}
}
}
switch ($document->getAttribute('$collection')) {
case OldDatabase::SYSTEM_COLLECTION_PLATFORMS:
$projectId = $this->getProjectIdFromReadPermissions($document);
/**
* Set Project ID
*/
if ($document->getAttribute('projectId') === null) {
$document->setAttribute('projectId', $projectId);
}
/**
* Set empty key and store if null
*/
if ($document->getAttribute('key') === null) {
$document->setAttribute('key', '');
}
if ($document->getAttribute('store') === null) {
$document->setAttribute('store', '');
}
/**
* Reset Permissions
*/
$document->setAttribute('$read', ['role:all']);
$document->setAttribute('$write', ['role:all']);
break;
case OldDatabase::SYSTEM_COLLECTION_DOMAINS:
$projectId = $this->getProjectIdFromReadPermissions($document);
/**
* Set Project ID
*/
if ($document->getAttribute('projectId') === null) {
$document->setAttribute('projectId', $projectId);
}
/**
* Set empty verification if null
*/
if ($document->getAttribute('verification') === null) {
$document->setAttribute('verification', false);
}
/**
* Reset Permissions
*/
$document->setAttribute('$read', ['role:all']);
$document->setAttribute('$write', ['role:all']);
break;
case OldDatabase::SYSTEM_COLLECTION_KEYS:
case OldDatabase::SYSTEM_COLLECTION_WEBHOOKS:
$projectId = $this->getProjectIdFromReadPermissions($document);
/**
* Set Project ID
*/
if ($document->getAttribute('projectId') === null) {
$document->setAttribute('projectId', $projectId);
}
/**
* Reset Permissions
*/
$document->setAttribute('$read', ['role:all']);
$document->setAttribute('$write', ['role:all']);
break;
case OldDatabase::SYSTEM_COLLECTION_USERS:
/**
* Set deleted attribute to false
*/
if ($document->getAttribute('deleted') === null) {
$document->setAttribute('deleted', false);
}
/**
* Remove deprecated user status 0 and replace with boolean.
*/
if ($document->getAttribute('status') === 2) {
$document->setAttribute('status', false);
} else {
$document->setAttribute('status', true);
}
/**
* Set default values for arrays if not set.
*/
if (empty($document->getAttribute('prefs', []))) {
$document->setAttribute('prefs', []);
}
if (empty($document->getAttribute('sessions', []))) {
$document->setAttribute('sessions', []);
}
if (empty($document->getAttribute('tokens', []))) {
$document->setAttribute('tokens', []);
}
if (empty($document->getAttribute('memberships', []))) {
$document->setAttribute('memberships', []);
}
/**
* Replace user:{self} with user:USER_ID
*/
$write = $document->getWrite();
$document->setAttribute('$write', str_replace('user:{self}', "user:{$document->getId()}", $write));
break;
case OldDatabase::SYSTEM_COLLECTION_FILES:
/**
* Migrating breakind changes on Files.
*/
if (!empty($document->getAttribute('fileOpenSSLVersion', null))) {
$document
->setAttribute('openSSLVersion', $document->getAttribute('fileOpenSSLVersion'))
->removeAttribute('fileOpenSSLVersion');
}
if (!empty($document->getAttribute('fileOpenSSLCipher', null))) {
$document
->setAttribute('openSSLCipher', $document->getAttribute('fileOpenSSLCipher'))
->removeAttribute('fileOpenSSLCipher');
}
if (!empty($document->getAttribute('fileOpenSSLTag', null))) {
$document
->setAttribute('openSSLTag', $document->getAttribute('fileOpenSSLTag'))
->removeAttribute('fileOpenSSLTag');
}
if (!empty($document->getAttribute('fileOpenSSLIV', null))) {
$document
->setAttribute('openSSLIV', $document->getAttribute('fileOpenSSLIV'))
->removeAttribute('fileOpenSSLIV');
}
/**
* Remove deprecated attributes.
*/
$document->removeAttribute('folderId');
$document->removeAttribute('token');
break;
}
return $document;
}
/**
* Migrates $permissions to independent $read and $write.
* @param Document $document
* @return Document
*/
protected function migratePermissions(Document $document): Document
{
if ($document->isSet('$permissions')) {
$permissions = $document->getAttribute('$permissions', []);
$read = $this->migrateWildcardPermissions($permissions['read'] ?? []);
$write = $this->migrateWildcardPermissions($permissions['write'] ?? []);
$document->setAttribute('$read', $read);
$document->setAttribute('$write', $write);
$document->removeAttribute('$permissions');
}
return $document;
}
/**
* Takes a permissions array and replaces wildcard * with role:all.
* @param array $permissions
* @return array
*/
protected function migrateWildcardPermissions(array $permissions): array
{
return array_map(function ($permission) {
if ($permission === '*') return 'role:all';
return $permission;
}, $permissions);
}
/**
* Get new collection attributes from old collection rules.
* @param OldDocument $collection
* @return array
*/
protected function getCollectionAttributes(OldDocument $collection): array
{
$attributes = [];
foreach ($collection->getAttribute('rules', []) as $key => $value) {
$collectionId = $collection->getId();
$id = $value['key'];
$array = $value['array'] ?? false;
$required = $value['required'] ?? false;
$default = $value['default'] ?? null;
$default = match ($value['type']) {
OldDatabase::SYSTEM_VAR_TYPE_NUMERIC => floatval($default),
default => $default
};
$type = match ($value['type']) {
OldDatabase::SYSTEM_VAR_TYPE_TEXT => Database::VAR_STRING,
OldDatabase::SYSTEM_VAR_TYPE_EMAIL => Database::VAR_STRING,
OldDatabase::SYSTEM_VAR_TYPE_DOCUMENT => Database::VAR_STRING,
OldDatabase::SYSTEM_VAR_TYPE_IP => Database::VAR_STRING,
OldDatabase::SYSTEM_VAR_TYPE_URL => Database::VAR_STRING,
OldDatabase::SYSTEM_VAR_TYPE_WILDCARD => Database::VAR_STRING,
OldDatabase::SYSTEM_VAR_TYPE_NUMERIC => Database::VAR_FLOAT,
OldDatabase::SYSTEM_VAR_TYPE_BOOLEAN => Database::VAR_BOOLEAN,
default => Database::VAR_STRING
};
$size = $type === Database::VAR_STRING ? 65_535 : 0; // Max size of text in MariaDB
$attributes[$key] = [
'$collection' => $collectionId,
'$id' => $id,
'type' => $type,
'size' => $size,
'required' => $required,
'default' => $default,
'array' => $array,
'signed' => true,
'filters' => []
];
if ($type === Database::VAR_FLOAT) {
$attributes[$key]['format'] = APP_DATABASE_ATTRIBUTE_FLOAT_RANGE;
$attributes[$key]['formatOptions'] = [];
$attributes[$key]['formatOptions']['min'] = -PHP_FLOAT_MAX;
$attributes[$key]['formatOptions']['max'] = PHP_FLOAT_MAX;
}
}
return $attributes;
}
/**
* @param Document $document
* @return string|null
* @throws Exception
*/
protected function getProjectIdFromReadPermissions(Document $document): string|null
{
$readPermissions = $document->getRead();
$teamId = str_replace('team:', '', reset($readPermissions));
return $this->oldConsoleDB->getCollectionFirst([
'filters' => [
'$collection=' . OldDatabase::SYSTEM_COLLECTION_PROJECTS,
'teamId=' . $teamId
]
])->getId();
}
}