2021-01-14 05:51:02 +13:00
|
|
|
<?php
|
|
|
|
|
|
|
|
namespace Appwrite\Migration;
|
|
|
|
|
2021-01-25 21:26:52 +13:00
|
|
|
use Swoole\Runtime;
|
2022-01-19 00:05:04 +13:00
|
|
|
use Utopia\Database\Document;
|
|
|
|
use Utopia\Database\Database;
|
2021-01-15 01:05:49 +13:00
|
|
|
use Utopia\CLI\Console;
|
2022-01-19 00:05:04 +13:00
|
|
|
use Utopia\Config\Config;
|
2022-02-07 12:24:49 +13:00
|
|
|
use Exception;
|
2022-05-14 00:49:31 +12:00
|
|
|
use Utopia\App;
|
|
|
|
use Utopia\Database\Validator\Authorization;
|
2021-01-14 05:51:02 +13:00
|
|
|
|
|
|
|
abstract class Migration
|
|
|
|
{
|
2021-01-19 03:43:55 +13:00
|
|
|
/**
|
|
|
|
* @var int
|
|
|
|
*/
|
2022-05-19 21:49:48 +12:00
|
|
|
protected int $limit = 250;
|
2021-12-09 06:50:04 +13:00
|
|
|
|
|
|
|
/**
|
2022-01-19 00:05:04 +13:00
|
|
|
* @var Document
|
2021-12-09 06:50:04 +13:00
|
|
|
*/
|
2022-01-19 00:05:04 +13:00
|
|
|
protected Document $project;
|
2021-01-15 03:09:48 +13:00
|
|
|
|
2021-01-19 03:43:55 +13:00
|
|
|
/**
|
2022-01-19 00:05:04 +13:00
|
|
|
* @var Database
|
2021-01-19 03:43:55 +13:00
|
|
|
*/
|
2022-01-19 00:05:04 +13:00
|
|
|
protected Database $projectDB;
|
2021-01-19 03:43:55 +13:00
|
|
|
|
|
|
|
/**
|
2022-01-19 00:05:04 +13:00
|
|
|
* @var Database
|
2021-01-19 03:43:55 +13:00
|
|
|
*/
|
2022-01-19 00:05:04 +13:00
|
|
|
protected Database $consoleDB;
|
2021-01-15 01:05:49 +13:00
|
|
|
|
2021-07-02 21:09:02 +12:00
|
|
|
/**
|
|
|
|
* @var array
|
|
|
|
*/
|
|
|
|
public static array $versions = [
|
2022-01-19 00:05:04 +13:00
|
|
|
'0.13.0' => 'V12',
|
2022-03-04 03:14:22 +13:00
|
|
|
'0.13.1' => 'V12',
|
2022-03-08 00:33:09 +13:00
|
|
|
'0.13.2' => 'V12',
|
2022-03-16 01:19:04 +13:00
|
|
|
'0.13.3' => 'V12',
|
2022-03-24 01:05:57 +13:00
|
|
|
'0.13.4' => 'V12',
|
2022-05-09 06:53:20 +12:00
|
|
|
'0.14.0' => 'V13',
|
2022-05-19 03:09:06 +12:00
|
|
|
'0.14.1' => 'V13',
|
2021-07-02 21:09:02 +12:00
|
|
|
];
|
|
|
|
|
2022-02-18 07:24:50 +13:00
|
|
|
/**
|
|
|
|
* @var array
|
|
|
|
*/
|
|
|
|
protected array $collections;
|
|
|
|
|
|
|
|
public function __construct()
|
|
|
|
{
|
2022-05-14 00:49:31 +12:00
|
|
|
Authorization::disable();
|
|
|
|
Authorization::setDefaultStatus(false);
|
2022-02-18 07:24:50 +13:00
|
|
|
$this->collections = array_merge([
|
|
|
|
'_metadata' => [
|
2022-05-14 00:49:31 +12:00
|
|
|
'$id' => '_metadata',
|
|
|
|
'$collection' => Database::METADATA
|
2022-02-18 07:24:50 +13:00
|
|
|
],
|
|
|
|
'audit' => [
|
2022-05-14 00:49:31 +12:00
|
|
|
'$id' => 'audit',
|
|
|
|
'$collection' => Database::METADATA
|
2022-02-18 07:24:50 +13:00
|
|
|
],
|
|
|
|
'abuse' => [
|
2022-05-14 00:49:31 +12:00
|
|
|
'$id' => 'abuse',
|
|
|
|
'$collection' => Database::METADATA
|
2022-02-18 07:24:50 +13:00
|
|
|
]
|
|
|
|
], Config::getParam('collections', []));
|
|
|
|
}
|
|
|
|
|
2021-01-15 01:05:49 +13:00
|
|
|
/**
|
|
|
|
* Set project for migration.
|
2021-10-06 20:53:33 +13:00
|
|
|
*
|
2022-01-19 00:05:04 +13:00
|
|
|
* @param Document $project
|
|
|
|
* @param Database $projectDB
|
|
|
|
* @param Database $oldConsoleDB
|
2021-10-06 20:53:33 +13:00
|
|
|
*
|
2021-12-09 06:50:04 +13:00
|
|
|
* @return self
|
2021-01-15 01:05:49 +13:00
|
|
|
*/
|
2022-01-19 00:05:04 +13:00
|
|
|
public function setProject(Document $project, Database $projectDB, Database $consoleDB): self
|
2021-01-15 01:05:49 +13:00
|
|
|
{
|
|
|
|
$this->project = $project;
|
2022-01-19 00:05:04 +13:00
|
|
|
$this->projectDB = $projectDB;
|
2022-02-18 07:24:50 +13:00
|
|
|
$this->projectDB->setNamespace('_' . $this->project->getId());
|
2021-12-09 06:50:04 +13:00
|
|
|
|
2022-01-19 00:05:04 +13:00
|
|
|
$this->consoleDB = $consoleDB;
|
2021-12-09 06:50:04 +13:00
|
|
|
|
2021-01-15 01:05:49 +13:00
|
|
|
return $this;
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Iterates through every document.
|
2021-10-06 20:53:33 +13:00
|
|
|
*
|
2021-01-19 03:43:55 +13:00
|
|
|
* @param callable $callback
|
2021-01-15 01:05:49 +13:00
|
|
|
*/
|
2021-01-21 22:57:15 +13:00
|
|
|
public function forEachDocument(callable $callback): void
|
2021-01-15 01:05:49 +13:00
|
|
|
{
|
2022-01-19 00:05:04 +13:00
|
|
|
Runtime::enableCoroutine(SWOOLE_HOOK_ALL);
|
|
|
|
|
2022-02-18 07:24:50 +13:00
|
|
|
foreach ($this->collections as $collection) {
|
2022-05-14 00:49:31 +12:00
|
|
|
if ($collection['$collection'] !== Database::METADATA) return;
|
2022-01-19 00:05:04 +13:00
|
|
|
$sum = 0;
|
|
|
|
$nextDocument = null;
|
|
|
|
$collectionCount = $this->projectDB->count($collection['$id']);
|
|
|
|
Console::log('Migrating Collection ' . $collection['$id'] . ':');
|
|
|
|
|
|
|
|
do {
|
2022-05-19 21:49:48 +12:00
|
|
|
$start = microtime(true);
|
2022-01-19 00:05:04 +13:00
|
|
|
$documents = $this->projectDB->find($collection['$id'], limit: $this->limit, cursor: $nextDocument);
|
2022-05-19 21:49:48 +12:00
|
|
|
var_dump("Fetching documents took " . microtime(true) - $start);
|
2022-01-19 00:05:04 +13:00
|
|
|
$count = count($documents);
|
|
|
|
$sum += $count;
|
|
|
|
|
|
|
|
Console::log($sum . ' / ' . $collectionCount);
|
|
|
|
|
|
|
|
\Co\run(function (array $documents, callable $callback) {
|
|
|
|
foreach ($documents as $document) {
|
|
|
|
go(function (Document $document, callable $callback) {
|
|
|
|
if (empty($document->getId()) || empty($document->getCollection())) {
|
|
|
|
return;
|
2021-10-16 05:11:20 +13:00
|
|
|
}
|
2021-01-25 21:26:52 +13:00
|
|
|
|
2022-05-19 21:49:48 +12:00
|
|
|
$start = microtime(true);
|
|
|
|
|
2022-01-19 00:05:04 +13:00
|
|
|
$old = $document->getArrayCopy();
|
|
|
|
$new = call_user_func($callback, $document);
|
2022-05-19 21:49:48 +12:00
|
|
|
var_dump("migration took " . microtime(true) - $start);
|
|
|
|
|
|
|
|
$start = microtime(true);
|
|
|
|
$diff = !$this->hasDifference($new->getArrayCopy(), $old);
|
|
|
|
var_dump("Diff took " . microtime(true) - $start);
|
2022-01-19 00:05:04 +13:00
|
|
|
|
2022-05-19 21:49:48 +12:00
|
|
|
if (!$diff) {
|
2022-01-19 00:05:04 +13:00
|
|
|
return;
|
|
|
|
}
|
2021-01-25 21:26:52 +13:00
|
|
|
|
2022-01-19 00:05:04 +13:00
|
|
|
try {
|
|
|
|
$new = $this->projectDB->updateDocument($document->getCollection(), $document->getId(), $document);
|
|
|
|
} catch (\Throwable $th) {
|
|
|
|
Console::error('Failed to update document: ' . $th->getMessage());
|
|
|
|
return;
|
2021-01-25 21:26:52 +13:00
|
|
|
|
2022-01-19 00:05:04 +13:00
|
|
|
if ($document && $new->getId() !== $document->getId()) {
|
|
|
|
throw new Exception('Duplication Error');
|
|
|
|
}
|
2021-01-25 21:26:52 +13:00
|
|
|
}
|
2022-01-19 00:05:04 +13:00
|
|
|
}, $document, $callback);
|
|
|
|
}
|
|
|
|
}, $documents, $callback);
|
|
|
|
|
|
|
|
if ($count !== $this->limit) {
|
|
|
|
$nextDocument = null;
|
|
|
|
} else {
|
|
|
|
$nextDocument = end($documents);
|
2021-01-15 01:05:49 +13:00
|
|
|
}
|
2022-01-19 00:05:04 +13:00
|
|
|
} while (!is_null($nextDocument));
|
2021-01-15 01:05:49 +13:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-12-09 06:50:04 +13:00
|
|
|
/**
|
|
|
|
* Checks 2 arrays for differences.
|
|
|
|
*
|
|
|
|
* @param array $array1
|
|
|
|
* @param array $array2
|
2022-05-19 21:29:01 +12:00
|
|
|
* @return bool
|
2021-12-09 06:50:04 +13:00
|
|
|
*/
|
2022-05-19 21:29:01 +12:00
|
|
|
public function hasDifference(array $array1, array $array2): bool
|
2021-10-06 20:53:33 +13:00
|
|
|
{
|
2022-05-19 21:29:01 +12:00
|
|
|
foreach ($array1 as $key => $value) {
|
|
|
|
if (is_array($value)) {
|
|
|
|
if (!isset($array2[$key]) || !is_array($array2[$key])) {
|
|
|
|
return true;
|
|
|
|
} else {
|
|
|
|
$new_diff = $this->hasDifference($value, $array2[$key]);
|
|
|
|
if ($new_diff) {
|
|
|
|
return true;
|
|
|
|
}
|
2021-04-16 19:08:42 +12:00
|
|
|
}
|
2022-05-19 21:29:01 +12:00
|
|
|
} elseif (!array_key_exists($key, $array2) || $array2[$key] !== $value) {
|
|
|
|
return true;
|
2021-04-16 19:08:42 +12:00
|
|
|
}
|
|
|
|
}
|
2021-10-16 05:11:20 +13:00
|
|
|
|
2022-05-19 21:29:01 +12:00
|
|
|
return false;
|
2021-04-16 19:08:42 +12:00
|
|
|
}
|
|
|
|
|
2022-05-14 00:49:31 +12:00
|
|
|
/**
|
|
|
|
* Creates colletion from the config collection.
|
|
|
|
*
|
|
|
|
* @param string $id
|
|
|
|
* @param string|null $name
|
|
|
|
* @return void
|
|
|
|
* @throws \Throwable
|
|
|
|
*/
|
|
|
|
protected function createCollection(string $id, string $name = null): void
|
|
|
|
{
|
|
|
|
$name ??= $id;
|
|
|
|
|
|
|
|
if (!$this->projectDB->exists(App::getEnv('_APP_DB_SCHEMA', 'appwrite'), $name)) {
|
|
|
|
$attributes = [];
|
|
|
|
$indexes = [];
|
|
|
|
$collection = $this->collections[$id];
|
|
|
|
|
|
|
|
foreach ($collection['attributes'] as $attribute) {
|
|
|
|
$attributes[] = new Document([
|
|
|
|
'$id' => $attribute['$id'],
|
|
|
|
'type' => $attribute['type'],
|
|
|
|
'size' => $attribute['size'],
|
|
|
|
'required' => $attribute['required'],
|
|
|
|
'signed' => $attribute['signed'],
|
|
|
|
'array' => $attribute['array'],
|
|
|
|
'filters' => $attribute['filters'],
|
|
|
|
]);
|
|
|
|
}
|
|
|
|
|
|
|
|
foreach ($collection['indexes'] as $index) {
|
|
|
|
$indexes[] = new Document([
|
|
|
|
'$id' => $index['$id'],
|
|
|
|
'type' => $index['type'],
|
|
|
|
'attributes' => $index['attributes'],
|
|
|
|
'lengths' => $index['lengths'],
|
|
|
|
'orders' => $index['orders'],
|
|
|
|
]);
|
|
|
|
}
|
|
|
|
|
|
|
|
try {
|
|
|
|
$this->projectDB->createCollection($name, $attributes, $indexes);
|
|
|
|
} catch (\Throwable $th) {
|
|
|
|
throw $th;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-01-15 01:05:49 +13:00
|
|
|
/**
|
|
|
|
* Executes migration for set project.
|
|
|
|
*/
|
|
|
|
abstract public function execute(): void;
|
2021-01-14 05:51:02 +13:00
|
|
|
}
|