From fb6455b783c554536aac864753301b840708e432 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Tue, 14 Nov 2023 14:43:33 +0100 Subject: [PATCH] Add script to patch missing repos documents --- Dockerfile | 1 + bin/patch-recreate-repositories-documents | 3 + .../patchRecreateRepositoriesDocuments.php | 141 ++++++++++++++++++ 3 files changed, 145 insertions(+) create mode 100644 bin/patch-recreate-repositories-documents create mode 100644 src/Appwrite/Platform/Tasks/patchRecreateRepositoriesDocuments.php diff --git a/Dockerfile b/Dockerfile index 33b143465..059c499bd 100755 --- a/Dockerfile +++ b/Dockerfile @@ -100,6 +100,7 @@ RUN chmod +x /usr/local/bin/doctor && \ RUN chmod +x /usr/local/bin/hamster && \ chmod +x /usr/local/bin/volume-sync && \ chmod +x /usr/local/bin/patch-delete-schedule-updated-at-attribute && \ + chmod +x /usr/local/bin/patch-recreate-repositories-documents && \ chmod +x /usr/local/bin/patch-delete-project-collections && \ chmod +x /usr/local/bin/delete-orphaned-projects && \ chmod +x /usr/local/bin/clear-card-cache && \ diff --git a/bin/patch-recreate-repositories-documents b/bin/patch-recreate-repositories-documents new file mode 100644 index 000000000..8c6c4157f --- /dev/null +++ b/bin/patch-recreate-repositories-documents @@ -0,0 +1,3 @@ +#!/bin/sh + +php /usr/src/code/app/cli.php patch-recreate-repositories-documents $@ \ No newline at end of file diff --git a/src/Appwrite/Platform/Tasks/patchRecreateRepositoriesDocuments.php b/src/Appwrite/Platform/Tasks/patchRecreateRepositoriesDocuments.php new file mode 100644 index 000000000..5749380d0 --- /dev/null +++ b/src/Appwrite/Platform/Tasks/patchRecreateRepositoriesDocuments.php @@ -0,0 +1,141 @@ +desc('Recreate missing repositories in consoleDB from projectDBs. They can be missing if you used Appwrite 1.4.10 or 1.4.11, and deleted a function.') + ->param('after', '', new Text(36), 'After cursor', true) + ->param('projectId', '', new Text(36), 'Select project to validate', true) + ->inject('dbForConsole') + ->inject('getProjectDB') + ->callback(fn ($after, $projectId, $dbForConsole, $getProjectDB) => $this->action($after, $projectId, $dbForConsole, $getProjectDB)); + } + + public function action($after, $projectId, Database $dbForConsole, callable $getProjectDB): void + { + Console::info("Starting the patch"); + + $startTime = microtime(true); + + if(!empty($projectId)) { + $project = $dbForConsole->getDocument('projects', $projectId); + $dbForProject = call_user_func($getProjectDB, $project); + $this->recreateRepositories($dbForConsole, $dbForProject, $project); + } else { + $queries = []; + if(!empty($after)) { + Console::info("Iterating remaining projects after project with ID {$after}"); + $project = $dbForConsole->getDocument('projects', $after); + $queries = [Query::cursorAfter($project)]; + } else { + Console::info("Iterating all projects"); + } + $this->foreachDocument($dbForConsole, 'projects', $queries, function(Document $project) use($getProjectDB, $dbForConsole){ + $dbForProject = call_user_func($getProjectDB, $project); + $this->recreateRepositories($dbForConsole, $dbForProject, $project); + }); + } + + $endTime = microtime(true); + $timeTaken = $endTime - $startTime; + + $hours = (int)($timeTaken / 3600); + $timeTaken -= $hours * 3600; + $minutes = (int)($timeTaken / 60); + $timeTaken -= $minutes * 60; + $seconds = (int)$timeTaken; + $milliseconds = ($timeTaken - $seconds) * 1000; + Console::info("Recreate patch completed in $hours h, $minutes m, $seconds s, $milliseconds mis ( total $timeTaken milliseconds)"); + } + + protected function foreachDocument(Database $database, string $collection, array $queries = [], callable $callback = null): void + { + $limit = 1000; + $results = []; + $sum = $limit; + $latestDocument = null; + + while ($sum === $limit) { + $newQueries = $queries; + + if ($latestDocument != null) { + array_unshift($newQueries, Query::cursorAfter($latestDocument)); + } + $newQueries[] = Query::limit($limit); + $results = $database->find($collection, $newQueries); + + if (empty($results)) { + return; + } + + $sum = count($results); + + foreach ($results as $document) { + if (is_callable($callback)) { + $callback($document); + } + } + $latestDocument = $results[array_key_last($results)]; + } + } + + public function recreateRepositories(Database $dbForConsole, Database $dbForProject, Document $project): void + { + $projectId = $project->getId(); + Console::log("Running patch for project {$projectId}"); + + $this->foreachDocument($dbForProject, 'functions', [], function(Document $function) use ($dbForConsole, $project) { + $isConnected = !empty($function->getAttribute('providerRepositoryId', '')); + + if($isConnected) { + $repository = $dbForConsole->getDocument('repositories', $function->getAttribute('repositoryId', '')); + + if($repository->isEmpty()) { + $projectId = $project->getId(); + $functionId = $function->getId(); + Console::success("Recreating repositories document for project ID {$projectId}, function ID {$functionId}"); + + $repository = $dbForConsole->createDocument('repositories', new Document([ + '$id' => ID::unique(), + '$permissions' => [ + Permission::read(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + 'installationId' => $function->getAttribute('installationId', ''), + 'installationInternalId' => $function->getAttribute('installationInternalId', ''), + 'projectId' => $project->getId(), + 'projectInternalId' => $project->getInternalId(), + 'providerRepositoryId' => $function->getAttribute('providerRepositoryId', ''), + 'resourceId' => $function->getId(), + 'resourceInternalId' => $function->getInternalId(), + 'resourceType' => 'function', + 'providerPullRequestIds' => [] + ])); + } + } + }); + } +}