CodeMirror/appwrite
1
0
Fork 0
mirror of synced 2024-10-02 10:16:27 +13:00
Code Issues Projects Releases Wiki Activity

Merge pull request #6001 from appwrite/feat-git-integration-update-migration

Browse source 
Update V19 Update Migrations for 1.4.x
This commit is contained in:
Jake Barnby 2023-08-22 19:40:50 -04:00 committed by GitHub
commit d7655bc6d3
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 639 additions and 224 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
app/cli.php
View file

@ -58,7 +58,7 @@ CLI::setResource('dbForConsole', function ($pools, $cache) {
->getResource();
$dbForConsole = new Database($dbAdapter, $cache);
$dbForConsole->setNamespace('console');
$dbForConsole->setNamespace('_console');
// Ensure tables exist
$collections = Config::getParam('collections', [])['console'];
@ -74,7 +74,7 @@ CLI::setResource('dbForConsole', function ($pools, $cache) {
$pools->get('console')->reclaim();
sleep($sleep);
}
} while ($attempts < $maxAttempts);
} while ($attempts < $maxAttempts && !$ready);
if (!$ready) {
throw new Exception("Console is not ready yet. Please try again later.");

4
app/init.php
View file

@ -108,7 +108,7 @@ const APP_LIMIT_LIST_DEFAULT = 25; // Default maximum number of items to return
const APP_KEY_ACCCESS = 24 * 60 * 60; // 24 hours
const APP_USER_ACCCESS = 24 * 60 * 60; // 24 hours
const APP_CACHE_UPDATE = 24 * 60 * 60; // 24 hours
const APP_CACHE_BUSTER = 506;
const APP_CACHE_BUSTER = 507;
const APP_VERSION_STABLE = '1.4.0';
const APP_DATABASE_ATTRIBUTE_EMAIL = 'email';
const APP_DATABASE_ATTRIBUTE_ENUM = 'enum';
@ -1116,7 +1116,7 @@ App::setResource('dbForConsole', function (Group $pools, Cache $cache) {
$database = new Database($dbAdapter, $cache);
$database->setNamespace('console');
$database->setNamespace('_console');
return $database;
}, ['pools', 'cache']);

2
app/realtime.php
View file

@ -47,7 +47,7 @@ function getConsoleDB(): Database
$database = new Database($dbAdapter, getCache());
$database->setNamespace('console');
$database->setNamespace('_console');
return $database;
}

2
app/worker.php
View file

@ -37,7 +37,7 @@ Server::setResource('dbForConsole', function (Cache $cache, Registry $register)
;
$adapter = new Database($database, $cache);
$adapter->setNamespace('console');
$adapter->setNamespace('_console');
return $adapter;
}, ['cache', 'register']);

83
src/Appwrite/Migration/Migration.php
View file

@ -77,7 +77,11 @@ abstract class Migration
Authorization::disable();
Authorization::setDefaultStatus(false);
$this->collections = array_merge([
$this->collections = Config::getParam('collections', []);
$projectCollections = $this->collections['projects'];
$this->collections['projects'] = array_merge([
'_metadata' => [
'$id' => ID::custom('_metadata'),
'$collection' => Database::METADATA
@ -90,7 +94,7 @@ abstract class Migration
'$id' => ID::custom('abuse'),
'$collection' => Database::METADATA
]
], Config::getParam('collections', []));
], $projectCollections);
}
/**
@ -131,7 +135,14 @@ abstract class Migration
*/
public function forEachDocument(callable $callback): void
{
foreach ($this->collections as $collection) {
$internalProjectId = $this->project->getInternalId();
$collections = match ($internalProjectId) {
'console' => $this->collections['console'],
default => $this->collections['projects'],
};
foreach ($collections as $collection) {
if ($collection['$collection'] !== Database::METADATA) {
continue;
}
@ -148,12 +159,12 @@ abstract class Migration
$old = $document->getArrayCopy();
$new = call_user_func($callback, $document);
if (is_null($new) || !self::hasDifference($new->getArrayCopy(), $old)) {
if (is_null($new) || $new->getArrayCopy() == $old) {
return;
}
try {
$new = $this->projectDB->updateDocument($document->getCollection(), $document->getId(), $document);
$this->projectDB->updateDocument($document->getCollection(), $document->getId(), $document);
} catch (\Throwable $th) {
Console::error('Failed to update document: ' . $th->getMessage());
return;
@ -199,32 +210,6 @@ abstract class Migration
} while (!is_null($nextDocument));
}
/**
* Checks 2 arrays for differences.
*
* @param array $array1
* @param array $array2
* @return bool
*/
public static function hasDifference(array $array1, array $array2): bool
{
foreach ($array1 as $key => $value) {
if (is_array($value)) {
if (!isset($array2[$key]) || !is_array($array2[$key])) {
return true;
} else {
if (self::hasDifference($value, $array2[$key])) {
return true;
}
}
} elseif (!array_key_exists($key, $array2) || $array2[$key] !== $value) {
return true;
}
}
return false;
}
/**
* Creates colletion from the config collection.
*
@ -237,10 +222,15 @@ abstract class Migration
{
$name ??= $id;
$collectionType = match ($this->project->getInternalId()) {
'console' => 'console',
default => 'projects',
};
if (!$this->projectDB->exists(App::getEnv('_APP_DB_SCHEMA', 'appwrite'), $name)) {
$attributes = [];
$indexes = [];
$collection = $this->collections[$id];
$collection = $this->collections[$collectionType][$id];
foreach ($collection['attributes'] as $attribute) {
$attributes[] = new Document([
@ -286,10 +276,22 @@ abstract class Migration
public function createAttributeFromCollection(Database $database, string $collectionId, string $attributeId, string $from = null): void
{
$from ??= $collectionId;
$collection = Config::getParam('collections', [])[$from] ?? null;
if (is_null($collection)) {
throw new Exception("Collection {$collectionId} not found");
$collectionType = match ($this->project->getInternalId()) {
'console' => 'console',
default => 'projects',
};
if ($from === 'files') {
$collectionType = 'buckets';
}
$collection = $this->collections[$collectionType][$from] ?? null;
if (is_null($collection)) {
throw new Exception("Collection {$from} not found");
}
$attributes = $collection['attributes'];
$attributeKey = array_search($attributeId, array_column($attributes, '$id'));
@ -332,17 +334,24 @@ abstract class Migration
public function createIndexFromCollection(Database $database, string $collectionId, string $indexId, string $from = null): void
{
$from ??= $collectionId;
$collection = Config::getParam('collections', [])[$collectionId] ?? null;
$collectionType = match ($this->project->getInternalId()) {
'console' => 'console',
default => 'projects',
};
$collection = $this->collections[$collectionType][$from] ?? null;
if (is_null($collection)) {
throw new Exception("Collection {$collectionId} not found");
}
$indexes = $collection['indexes'];
$indexKey = array_search($indexId, array_column($indexes, '$id'));
if ($indexKey === false) {
throw new Exception("Attribute {$indexId} not found");
throw new Exception("Index {$indexId} not found");
}
$index = $indexes[$indexKey];

642
src/Appwrite/Migration/Version/V19.php
View file

@ -2,12 +2,14 @@
namespace Appwrite\Migration\Version;
use Appwrite\Auth\Auth;
use Utopia\Config\Config;
use Appwrite\Migration\Migration;
use Utopia\App;
use Utopia\CLI\Console;
use Utopia\Config\Config;
use Utopia\Database\Database;
use Utopia\Database\DateTime;
use Utopia\Database\Document;
use Utopia\Database\Exception;
class V19 extends Migration
{
@ -28,12 +30,6 @@ class V19 extends Migration
Console::log('Migrating Project: ' . $this->project->getAttribute('name') . ' (' . $this->project->getId() . ')');
$this->projectDB->setNamespace("_{$this->project->getInternalId()}");
$this->alterPermissionIndex('_metadata');
$this->alterUidType('_metadata');
Console::info('Migrating Databases');
$this->migrateDatabases();
Console::info('Migrating Collections');
$this->migrateCollections();
@ -42,29 +38,29 @@ class V19 extends Migration
Console::info('Migrating Documents');
$this->forEachDocument([$this, 'fixDocument']);
Console::log('Cleaning Up Collections');
$this->cleanCollections();
}
/**
* Migrate all Databases.
* Migrating all Bucket tables.
*
* @return void
* @throws \Exception
* @throws \PDOException
*/
private function migrateDatabases(): void
protected function migrateBuckets(): void
{
foreach ($this->documentsIterator('databases') as $database) {
Console::log("Migrating Collections of {$database->getId()} ({$database->getAttribute('name')})");
foreach ($this->documentsIterator('buckets') as $bucket) {
$id = "bucket_{$bucket->getInternalId()}";
Console::log("Migrating Bucket {$id} {$bucket->getId()} ({$bucket->getAttribute('name')})");
$databaseTable = "database_{$database->getInternalId()}";
$this->alterPermissionIndex($databaseTable);
$this->alterUidType($databaseTable);
foreach ($this->documentsIterator($databaseTable) as $collection) {
$collectionTable = "{$databaseTable}_collection_{$collection->getInternalId()}";
Console::log("Migrating Collections of {$collectionTable} {$collection->getId()} ({$collection->getAttribute('name')})");
$this->alterPermissionIndex($collectionTable);
$this->alterUidType($collectionTable);
try {
$this->createAttributeFromCollection($this->projectDB, $id, 'bucketInternalId', 'files');
$this->projectDB->deleteCachedCollection($id);
} catch (\Throwable $th) {
Console::warning("'bucketInternalId' from {$id}: {$th->getMessage()}");
}
}
}
@ -73,36 +69,457 @@ class V19 extends Migration
* Migrate all Collections.
*
* @return void
* @throws \Throwable
* @throws Exception
*/
private function migrateCollections(): void
{
foreach ($this->collections as $collection) {
$internalProjectId = $this->project->getInternalId();
$collectionType = match ($internalProjectId) {
'console' => 'console',
default => 'projects',
};
$collections = $this->collections[$collectionType];
foreach ($collections as $collection) {
$id = $collection['$id'];
if ($id === 'schedules' && $internalProjectId === 'console') {
continue;
}
Console::log("Migrating Collection \"{$id}\"");
$this->projectDB->setNamespace("_{$this->project->getInternalId()}");
$this->projectDB->setNamespace("_$internalProjectId");
switch ($id) {
case 'projects':
case '_metadata':
$this->createCollection('identities');
$this->createCollection('migrations');
// Holding off on this until a future release
// $this->createCollection('statsLogger');
break;
case 'attributes':
case 'indexes':
try {
/**
* Create 'passwordHistory' attribute
*/
$this->createAttributeFromCollection($this->projectDB, $id, 'smtp');
$this->createAttributeFromCollection($this->projectDB, $id, 'templates');
$this->projectDB->deleteCachedCollection($id);
$this->projectDB->updateAttribute($id, 'databaseInternalId', required: true);
} catch (\Throwable $th) {
Console::warning("'SMTP and Templates' from {$id}: {$th->getMessage()}");
Console::warning("'databaseInternalId' from {$id}: {$th->getMessage()}");
}
try {
$this->projectDB->updateAttribute($id, 'collectionInternalId', required: true);
} catch (\Throwable $th) {
Console::warning("'collectionInternalId' from {$id}: {$th->getMessage()}");
}
try {
$this->createAttributeFromCollection($this->projectDB, $id, 'error');
} catch (\Throwable $th) {
Console::warning("'error' from {$id}: {$th->getMessage()}");
}
$this->projectDB->deleteCachedCollection($id);
break;
case 'buckets':
// Recreate indexes so they're the right size
$indexesToDelete = [
'_key_name',
];
foreach ($indexesToDelete as $index) {
try {
$this->projectDB->deleteIndex($id, $index);
} catch (\Throwable $th) {
Console::warning("'$index' from {$id}: {$th->getMessage()}");
}
}
$indexesToCreate = [
...$indexesToDelete
];
foreach ($indexesToCreate as $index) {
try {
$this->createIndexFromCollection($this->projectDB, $id, $index);
} catch (\Throwable $th) {
Console::warning("'$index' from {$id}: {$th->getMessage()}");
}
}
$this->projectDB->deleteCachedCollection($id);
break;
case 'builds':
try {
$this->createAttributeFromCollection($this->projectDB, $id, 'deploymentInternalId');
} catch (\Throwable $th) {
Console::warning("'deploymentInternalId' from {$id}: {$th->getMessage()}");
}
try {
$this->createAttributeFromCollection($this->projectDB, $id, 'logs');
} catch (\Throwable $th) {
Console::warning("'logs' from {$id}: {$th->getMessage()}");
}
$this->projectDB->deleteCachedCollection($id);
break;
case 'certificates':
try {
$this->projectDB->renameAttribute($id, 'log', 'logs');
} catch (\Throwable $th) {
Console::warning("'errors' from {$id}: {$th->getMessage()}");
}
try {
$this->projectDB->updateAttribute($id, 'logs', size: 1000000);
} catch (\Throwable $th) {
Console::warning("'errors' from {$id}: {$th->getMessage()}");
}
$this->projectDB->deleteCachedCollection($id);
break;
case 'databases':
try {
$this->createAttributeFromCollection($this->projectDB, $id, 'enabled');
} catch (\Throwable $th) {
Console::warning("'enabled' from {$id}: {$th->getMessage()}");
}
$this->projectDB->deleteCachedCollection($id);
break;
case 'deployments':
$attributesToCreate = [
'resourceInternalId',
'buildInternalId',
'type',
];
foreach ($attributesToCreate as $attribute) {
try {
$this->createAttributeFromCollection($this->projectDB, $id, $attribute);
} catch (\Throwable $th) {
Console::warning("$attribute from {$id}: {$th->getMessage()}");
}
}
// Recreate indexes so they're the right size
$indexesToDelete = [
'_key_entrypoint',
'_key_resource',
'_key_resource_type',
'_key_buildId',
];
foreach ($indexesToDelete as $index) {
try {
$this->projectDB->deleteIndex($id, $index);
} catch (\Throwable $th) {
Console::warning("'$index' from {$id}: {$th->getMessage()}");
}
}
$indexesToCreate = [
'_key_resource',
'_key_resource_type',
'_key_buildId',
];
foreach ($indexesToCreate as $index) {
try {
$this->createIndexFromCollection($this->projectDB, $id, $index);
} catch (\Throwable $th) {
Console::warning("'$index' from {$id}: {$th->getMessage()}");
}
}
$this->projectDB->deleteCachedCollection($id);
break;
case 'executions':
try {
$this->createAttributeFromCollection($this->projectDB, $id, 'functionInternalId');
} catch (\Throwable $th) {
Console::warning("'functionInternalId' from {$id}: {$th->getMessage()}");
}
try {
$this->createAttributeFromCollection($this->projectDB, $id, 'deploymentInternalId');
} catch (\Throwable $th) {
Console::warning("'deploymentInternalId' from {$id}: {$th->getMessage()}");
}
try {
$this->projectDB->renameAttribute($id, 'stderr', 'errors');
} catch (\Throwable $th) {
Console::warning("'errors' from {$id}: {$th->getMessage()}");
}
try {
$this->projectDB->renameAttribute($id, 'stdout', 'logs');
} catch (\Throwable $th) {
Console::warning("'logs' from {$id}: {$th->getMessage()}");
}
try {
$this->projectDB->renameAttribute($id, 'statusCode', 'responseStatusCode');
} catch (\Throwable $th) {
Console::warning("'responseStatusCode' from {$id}: {$th->getMessage()}");
}
$this->projectDB->deleteCachedCollection($id);
break;
case 'files':
// Recreate indexes so they're the right size
$indexesToDelete = [
'_key_name',
'_key_signature',
'_key_mimeType',
];
foreach ($indexesToDelete as $index) {
try {
$this->projectDB->deleteIndex($id, $index);
} catch (\Throwable $th) {
Console::warning("'$index' from {$id}: {$th->getMessage()}");
}
}
$indexesToCreate = $indexesToDelete;
foreach ($indexesToCreate as $index) {
try {
$this->createIndexFromCollection($this->projectDB, $id, $index);
} catch (\Throwable $th) {
Console::warning("'$index' from {$id}: {$th->getMessage()}");
}
}
$this->projectDB->deleteCachedCollection($id);
break;
case 'functions':
$attributesToCreate = [
'live',
'installationId',
'installationInternalId',
'providerRepositoryId',
'repositoryId',
'repositoryInternalId',
'providerBranch',
'providerRootDirectory',
'providerSilentMode',
'logging',
'deploymentInternalId',
'scheduleInternalId',
'scheduleId',
'version',
'entrypoint',
'commands',
];
foreach ($attributesToCreate as $attribute) {
try {
$this->createAttributeFromCollection($this->projectDB, $id, $attribute);
} catch (\Throwable $th) {
Console::warning("'$attribute' from {$id}: {$th->getMessage()}");
}
}
// Recreate indexes so they're the right size
$indexesToDelete = [
'_key_name',
'_key_runtime',
'_key_deployment',
];
foreach ($indexesToDelete as $index) {
try {
$this->projectDB->deleteIndex($id, $index);
} catch (\Throwable $th) {
Console::warning("'$index' from {$id}: {$th->getMessage()}");
}
}
$indexesToCreate = [
...$indexesToDelete,
'_key_installationId',
'_key_installationInternalId',
'_key_providerRepositoryId',
'_key_repositoryId',
'_key_repositoryInternalId',
];
foreach ($indexesToCreate as $index) {
try {
$this->createIndexFromCollection($this->projectDB, $id, $index);
} catch (\Throwable $th) {
Console::warning("'$index' from {$id}: {$th->getMessage()}");
}
}
$this->projectDB->deleteCachedCollection($id);
break;
case 'memberships':
try {
$this->projectDB->updateAttribute($id, 'teamInternalId', required: true);
} catch (\Throwable $th) {
Console::warning("'teamInternalId' from {$id}: {$th->getMessage()}");
}
$this->projectDB->deleteCachedCollection($id);
// Intentional fall through to update memberships.userInternalId
case 'sessions':
case 'tokens':
try {
$this->projectDB->updateAttribute($id, 'userInternalId', required: true);
} catch (\Throwable $th) {
Console::warning("'userInternalId' from {$id}: {$th->getMessage()}");
}
$this->projectDB->deleteCachedCollection($id);
break;
case 'domains':
case 'keys':
case 'platforms':
case 'webhooks':
try {
$this->projectDB->updateAttribute($id, 'projectInternalId', required: true);
} catch (\Throwable $th) {
Console::warning("'projectInternalId' from {$id}: {$th->getMessage()}");
}
$this->projectDB->deleteCachedCollection($id);
break;
case 'projects':
$attributesToCreate = [
'database',
'smtp',
'templates',
];
foreach ($attributesToCreate as $attribute) {
try {
$this->createAttributeFromCollection($this->projectDB, $id, $attribute);
} catch (\Throwable $th) {
Console::warning("'$attribute' from {$id}: {$th->getMessage()}");
Console::warning($th->getTraceAsString());
}
}
$this->projectDB->deleteCachedCollection($id);
break;
case 'stats':
try {
$this->projectDB->updateAttribute($id, 'value', signed: true);
} catch (\Throwable $th) {
Console::warning("'value' from {$id}: {$th->getMessage()}");
}
// Holding off on these until a future release
// try {
// $this->projectDB->deleteAttribute($id, 'type');
// $this->projectDB->deleteCachedCollection($id);
// } catch (\Throwable $th) {
// Console::warning("'type' from {$id}: {$th->getMessage()}");
// }
// try {
// $this->projectDB->deleteIndex($id, '_key_metric_period_time');
// $this->projectDB->deleteCachedCollection($id);
// } catch (\Throwable $th) {
// Console::warning("'_key_metric_period_time' from {$id}: {$th->getMessage()}");
// }
// try {
// $this->createIndexFromCollection($this->projectDB, $id, '_key_metric_period_time');
// $this->projectDB->deleteCachedCollection($id);
// } catch (\Throwable $th) {
// Console::warning("'_key_metric_period_time' from {$id}: {$th->getMessage()}");
// }
$this->projectDB->deleteCachedCollection($id);
break;
case 'users':
try {
$this->createAttributeFromCollection($this->projectDB, $id, 'labels');
} catch (\Throwable $th) {
Console::warning("'labels' from {$id}: {$th->getMessage()}");
}
try {
$this->createAttributeFromCollection($this->projectDB, $id, 'accessedAt');
} catch (\Throwable $th) {
Console::warning("'accessedAt' from {$id}: {$th->getMessage()}");
}
try {
$this->projectDB->updateAttribute($id, 'search', filters: ['userSearch']);
} catch (\Throwable $th) {
Console::warning("'search' from {$id}: {$th->getMessage()}");
}
try {
$this->createIndexFromCollection($this->projectDB, $id, '_key_accessedAt');
} catch (\Throwable $th) {
Console::warning("'_key_accessedAt' from {$id}: {$th->getMessage()}");
}
$this->projectDB->deleteCachedCollection($id);
break;
case 'variables':
try {
$this->projectDB->deleteIndex($id, '_key_function');
} catch (\Throwable $th) {
Console::warning("'_key_function' from {$id}: {$th->getMessage()}");
}
try {
$this->projectDB->deleteIndex($id, '_key_uniqueKey');
} catch (\Throwable $th) {
Console::warning("'_key_uniqueKey' from {$id}: {$th->getMessage()}");
}
try {
$this->createAttributeFromCollection($this->projectDB, $id, 'resourceType');
} catch (\Throwable $th) {
Console::warning("'resourceType' from {$id}: {$th->getMessage()}");
}
try {
$this->projectDB->renameAttribute($id, 'functionInternalId', 'resourceInternalId');
} catch (\Throwable $th) {
Console::warning("'resourceInternalId' from {$id}: {$th->getMessage()}");
}
try {
$this->projectDB->renameAttribute($id, 'functionId', 'resourceId');
} catch (\Throwable $th) {
Console::warning("'resourceId' from {$id}: {$th->getMessage()}");
}
$indexesToCreate = [
'_key_resourceInternalId',
'_key_resourceId_resourceType',
'_key_resourceType',
'_key_uniqueKey',
];
foreach ($indexesToCreate as $index) {
try {
$this->createIndexFromCollection($this->projectDB, $id, $index);
} catch (\Throwable $th) {
Console::warning("'$index' from {$id}: {$th->getMessage()}");
}
}
$this->projectDB->deleteCachedCollection($id);
break;
default:
break;
}
if (!in_array($id, ['files', 'collections'])) {
$this->alterPermissionIndex($id);
$this->alterUidType($id);
}
usleep(50000);
}
@ -117,62 +534,145 @@ class V19 extends Migration
protected function fixDocument(Document $document): Document
{
switch ($document->getCollection()) {
case 'attributes':
case 'indexes':
$status = $document->getAttribute('status', '');
if ($status === 'failed') {
$document->setAttribute('error', 'Unknown problem');
}
break;
case 'builds':
$deploymentId = $document->getAttribute('deploymentId');
$deployment = $this->projectDB->getDocument('deployments', $deploymentId);
$document->setAttribute('deploymentInternalId', $deployment->getInternalId());
$stdout = $document->getAttribute('stdout', '');
$stderr = $document->getAttribute('stderr', '');
$document->setAttribute('logs', $stdout . PHP_EOL . $stderr);
break;
case 'databases':
$document->setAttribute('enabled', true);
break;
case 'deployments':
$resourceId = $document->getAttribute('resourceId');
$function = $this->projectDB->getDocument('functions', $resourceId);
$document->setAttribute('resourceInternalId', $function->getInternalId());
$buildId = $document->getAttribute('buildId');
if (!empty($buildId)) {
$build = $this->projectDB->getDocument('builds', $buildId);
$document->setAttribute('buildInternalId', $build->getInternalId());
}
$document->setAttribute('type', 'manual');
break;
case 'executions':
$functionId = $document->getAttribute('functionId');
$function = $this->projectDB->getDocument('functions', $functionId);
$document->setAttribute('functionInternalId', $function->getInternalId());
$deploymentId = $document->getAttribute('deploymentId');
$deployment = $this->projectDB->getDocument('deployments', $deploymentId);
$document->setAttribute('deploymentInternalId', $deployment->getInternalId());
break;
case 'functions':
$document->setAttribute('live', true);
$document->setAttribute('logging', true);
$document->setAttribute('version', 'v2');
$deploymentId = $document->getAttribute('deployment');
if (!empty($deploymentId)) {
$deployment = $this->projectDB->getDocument('deployments', $deploymentId);
$document->setAttribute('deploymentInternalId', $deployment->getInternalId());
$document->setAttribute('entrypoint', $deployment->getAttribute('entrypoint'));
}
$schedule = $this->consoleDB->createDocument('schedules', new Document([
'region' => App::getEnv('_APP_REGION', 'default'), // Todo replace with projects region
'resourceType' => 'function',
'resourceId' => $document->getId(),
'resourceInternalId' => $document->getInternalId(),
'resourceUpdatedAt' => DateTime::now(),
'projectId' => $this->project->getId(),
'schedule' => $document->getAttribute('schedule'),
'active' => !empty($document->getAttribute('schedule')) && !empty($document->getAttribute('deployment')),
]));
$document->setAttribute('scheduleId', $schedule->getId());
$document->setAttribute('scheduleInternalId', $schedule->getInternalId());
break;
case 'projects':
/**
* Bump version number.
*/
$document->setAttribute('version', '1.4.0');
$databases = Config::getParam('pools-database', []);
$database = $databases[0];
$document->setAttribute('database', $database);
$document->setAttribute('smtp', []);
$document->setAttribute('templates', []);
break;
case 'rules':
$status = 'created';
if ($document->getAttribute('verification', false)) {
$status = 'verified';
}
$ruleDocument = new Document([
'projectId' => $this->project->getId(),
'projectInternalId' => $this->project->getInternalId(),
'domain' => $document->getAttribute('domain'),
'resourceType' => 'api',
'resourceInternalId' => '',
'resourceId' => '',
'status' => $status,
'certificateId' => $document->getAttribute('certificateId'),
]);
try {
$this->consoleDB->createDocument('rules', $ruleDocument);
} catch (\Throwable $th) {
Console::warning("Error migrating domain {$document->getAttribute('domain')}: {$th->getMessage()}");
}
break;
default:
break;
}
return $document;
}
protected function alterPermissionIndex($collectionName): void
private function cleanCollections(): void
{
try {
$table = "`{$this->projectDB->getDefaultDatabase()}`.`_{$this->project->getInternalId()}_{$collectionName}_perms`";
$this->pdo->prepare("
ALTER TABLE {$table}
DROP INDEX `_permission`,
ADD INDEX `_permission` (`_permission`, `_type`, `_document`);
")->execute();
$this->projectDB->deleteAttribute('projects', 'domains');
} catch (\Throwable $th) {
Console::warning($th->getMessage());
Console::warning("'domains' from projects: {$th->getMessage()}");
}
}
protected function alterUidType($collectionName): void
{
$this->projectDB->deleteCachedCollection('projects');
try {
$table = "`{$this->projectDB->getDefaultDatabase()}`.`_{$this->project->getInternalId()}_{$collectionName}`";
$this->pdo->prepare("
ALTER TABLE {$table}
CHANGE COLUMN `_uid` `_uid` VARCHAR(255) NOT NULL ;
")->execute();
$this->projectDB->deleteAttribute('functions', 'schedule');
} catch (\Throwable $th) {
Console::warning($th->getMessage());
Console::warning("'schedule' from functions: {$th->getMessage()}");
}
}
/**
* Migrating all Bucket tables.
*
* @return void
* @throws \Exception
* @throws \PDOException
*/
protected function migrateBuckets(): void
{
foreach ($this->documentsIterator('buckets') as $bucket) {
$id = "bucket_{$bucket->getInternalId()}";
Console::log("Migrating Bucket {$id} {$bucket->getId()} ({$bucket->getAttribute('name')})");
$this->alterPermissionIndex($id);
$this->alterUidType($id);
$this->projectDB->deleteCachedCollection('functions');
try {
$this->projectDB->deleteAttribute('builds', 'stderr');
} catch (\Throwable $th) {
Console::warning("'stderr' from builds: {$th->getMessage()}");
}
try {
$this->projectDB->deleteAttribute('builds', 'stdout');
} catch (\Throwable $th) {
Console::warning("'stdout' from builds: {$th->getMessage()}");
}
$this->projectDB->deleteCachedCollection('builds');
}
}

34
src/Appwrite/Platform/Tasks/Migrate.php
View file

@ -7,10 +7,10 @@ use Utopia\CLI\Console;
use Appwrite\Migration\Migration;
use Utopia\App;
use Utopia\Cache\Cache;
use Utopia\Cache\Adapter\Redis as RedisCache;
use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\Database\Query;
use Utopia\Database\Validator\Authorization;
use Utopia\Registry\Registry;
use Utopia\Validator\Text;
class Migrate extends Action
@ -26,20 +26,22 @@ class Migrate extends Action
->desc('Migrate Appwrite to new version')
/** @TODO APP_VERSION_STABLE needs to be defined */
->param('version', APP_VERSION_STABLE, new Text(8), 'Version to migrate to.', true)
->inject('register')
->callback(fn ($version, $register) => $this->action($version, $register));
->inject('cache')
->inject('dbForConsole')
->inject('getProjectDB')
->callback(fn ($version, $cache, $dbForConsole, $getProjectDB) => $this->action($version, $cache, $dbForConsole, $getProjectDB));
}
private function clearProjectsCache(Redis $redis, Document $project)
private function clearProjectsCache(Cache $cache, Document $project)
{
try {
$redis->del($redis->keys("cache-_{$project->getInternalId()}:*"));
$cache->purge("cache-_{$project->getInternalId()}:*");
} catch (\Throwable $th) {
Console::error('Failed to clear project ("' . $project->getId() . '") cache with error: ' . $th->getMessage());
}
}
public function action(string $version, Registry $register)
public function action(string $version, Cache $cache, Database $dbForConsole, callable $getProjectDB)
{
Authorization::disable();
if (!array_key_exists($version, Migration::$versions)) {
@ -52,14 +54,6 @@ class Migrate extends Action
Console::success('Starting Data Migration to version ' . $version);
$dbPool = $register->get('dbPool', true);
$redis = $register->get('cache', true);
$cache = new Cache(new RedisCache($redis));
$dbForConsole = $dbPool->getDB('console', $cache);
$dbForConsole->setNamespace('_project_console');
$console = $app->getResource('console');
$limit = 30;
@ -79,6 +73,7 @@ class Migrate extends Action
}
$class = 'Appwrite\\Migration\\Version\\' . Migration::$versions[$version];
/** @var Migration $migration */
$migration = new $class();
while (!empty($projects)) {
@ -90,11 +85,11 @@ class Migrate extends Action
continue;
}
$this->clearProjectsCache($redis, $project);
$this->clearProjectsCache($cache, $project);
try {
// TODO: Iterate through all project DBs
$projectDB = $dbPool->getDB($project->getId(), $cache);
$projectDB = $getProjectDB($project);
$migration
->setProject($project, $projectDB, $dbForConsole)
->execute();
@ -103,11 +98,11 @@ class Migrate extends Action
throw $th;
}
$this->clearProjectsCache($redis, $project);
$this->clearProjectsCache($cache, $project);
}
$sum = \count($projects);
$projects = $dbForConsole->find('projects', limit: $limit, offset: $offset);
$projects = $dbForConsole->find('projects', [Query::limit($limit), Query::offset($offset)]);
$offset = $offset + $limit;
$count = $count + $sum;
@ -115,7 +110,6 @@ class Migrate extends Action
Console::log('Migrated ' . $count . '/' . $totalProjects . ' projects...');
}
Swoole\Event::wait(); // Wait for Coroutines to finish
Console::success('Data Migration Completed');
}
}

4
src/Appwrite/Resque/Worker.php
View file

@ -223,7 +223,7 @@ abstract class Worker
if (isset(self::$databases[$databaseName])) {
$database = self::$databases[$databaseName];
$database->setNamespace('console');
$database->setNamespace('_console');
return $database;
}
@ -237,7 +237,7 @@ abstract class Worker
self::$databases[$databaseName] = $database;
$database->setNamespace('console');
$database->setNamespace('_console');
return $database;
}

88
tests/unit/Migration/MigrationTest.php
View file

@ -47,92 +47,4 @@ abstract class MigrationTest extends TestCase
$this->assertArrayHasKey(APP_VERSION_STABLE, Migration::$versions);
}
}
public function testHasDifference(): void
{
$this->assertFalse(Migration::hasDifference([], []));
$this->assertFalse(Migration::hasDifference([
'bool' => true,
'string' => 'abc',
'int' => 123,
'array' => ['a', 'b', 'c'],
'assoc' => [
'a' => true,
'b' => 'abc',
'c' => 123,
'd' => ['a', 'b', 'c']
]
], [
'bool' => true,
'string' => 'abc',
'int' => 123,
'array' => ['a', 'b', 'c'],
'assoc' => [
'a' => true,
'b' => 'abc',
'c' => 123,
'd' => ['a', 'b', 'c']
]
]));
$this->assertFalse(Migration::hasDifference([
'bool' => true,
'string' => 'abc',
'int' => 123,
'array' => ['a', 'b', 'c'],
'assoc' => [
'a' => true,
'b' => 'abc',
'c' => 123,
'd' => ['a', 'b', 'c']
]
], [
'string' => 'abc',
'assoc' => [
'a' => true,
'b' => 'abc',
'c' => 123,
'd' => ['a', 'b', 'c']
],
'int' => 123,
'array' => ['a', 'b', 'c'],
'bool' => true,
]));
$this->assertTrue(Migration::hasDifference([
'a' => true
], [
'b' => true
]));
$this->assertTrue(Migration::hasDifference([
'a' => 'true'
], [
'a' => true
]));
$this->assertTrue(Migration::hasDifference([
'a' => true
], [
'a' => false
]));
$this->assertTrue(Migration::hasDifference([
'nested' => [
'a' => true
]
], [
'nested' => []
]));
$this->assertTrue(Migration::hasDifference([
'assoc' => [
'bool' => true,
'string' => 'abc',
'int' => 123,
'array' => ['a', 'b', 'c']
]
], [
'nested' => [
'a' => true,
'int' => '123',
'array' => ['a', 'b', 'c']
]
]));
}
}