Merge pull request #7133 from appwrite/fix-repositories-collection
Repositories recreation script
This commit is contained in:
commit
29cd164448
7 changed files with 178 additions and 17 deletions
|
@ -100,6 +100,7 @@ RUN chmod +x /usr/local/bin/doctor && \
|
|||
RUN chmod +x /usr/local/bin/hamster && \
|
||||
chmod +x /usr/local/bin/volume-sync && \
|
||||
chmod +x /usr/local/bin/patch-delete-schedule-updated-at-attribute && \
|
||||
chmod +x /usr/local/bin/patch-recreate-repositories-documents && \
|
||||
chmod +x /usr/local/bin/patch-delete-project-collections && \
|
||||
chmod +x /usr/local/bin/delete-orphaned-projects && \
|
||||
chmod +x /usr/local/bin/clear-card-cache && \
|
||||
|
|
|
@ -242,12 +242,16 @@ App::post('/v1/functions')
|
|||
|
||||
// Git connect logic
|
||||
if (!empty($providerRepositoryId)) {
|
||||
$teamId = $project->getAttribute('teamId', '');
|
||||
|
||||
$repository = $dbForConsole->createDocument('repositories', new Document([
|
||||
'$id' => ID::unique(),
|
||||
'$permissions' => [
|
||||
Permission::read(Role::any()),
|
||||
Permission::update(Role::any()),
|
||||
Permission::delete(Role::any()),
|
||||
Permission::read(Role::team(ID::custom($teamId))),
|
||||
Permission::update(Role::team(ID::custom($teamId), 'owner')),
|
||||
Permission::update(Role::team(ID::custom($teamId), 'developer')),
|
||||
Permission::delete(Role::team(ID::custom($teamId), 'owner')),
|
||||
Permission::delete(Role::team(ID::custom($teamId), 'developer')),
|
||||
],
|
||||
'installationId' => $installation->getId(),
|
||||
'installationInternalId' => $installation->getInternalId(),
|
||||
|
|
|
@ -857,10 +857,10 @@ App::post('/v1/vcs/github/events')
|
|||
$github->initializeVariables($providerInstallationId, $privateKey, $githubAppId);
|
||||
|
||||
//find functionId from functions table
|
||||
$repositories = $dbForConsole->find('repositories', [
|
||||
$repositories = Authorization::skip(fn () => $dbForConsole->find('repositories', [
|
||||
Query::equal('providerRepositoryId', [$providerRepositoryId]),
|
||||
Query::limit(100),
|
||||
]);
|
||||
]));
|
||||
|
||||
// create new deployment only on push and not when branch is created
|
||||
if (!$providerBranchCreated) {
|
||||
|
@ -877,13 +877,13 @@ App::post('/v1/vcs/github/events')
|
|||
]);
|
||||
|
||||
foreach ($installations as $installation) {
|
||||
$repositories = $dbForConsole->find('repositories', [
|
||||
$repositories = Authorization::skip(fn () => $dbForConsole->find('repositories', [
|
||||
Query::equal('installationInternalId', [$installation->getInternalId()]),
|
||||
Query::limit(1000)
|
||||
]);
|
||||
]));
|
||||
|
||||
foreach ($repositories as $repository) {
|
||||
$dbForConsole->deleteDocument('repositories', $repository->getId());
|
||||
Authorization::skip(fn () => $dbForConsole->deleteDocument('repositories', $repository->getId()));
|
||||
}
|
||||
|
||||
$dbForConsole->deleteDocument('installations', $installation->getId());
|
||||
|
@ -915,10 +915,10 @@ App::post('/v1/vcs/github/events')
|
|||
$providerCommitAuthor = $commitDetails["commitAuthor"] ?? '';
|
||||
$providerCommitMessage = $commitDetails["commitMessage"] ?? '';
|
||||
|
||||
$repositories = $dbForConsole->find('repositories', [
|
||||
$repositories = Authorization::skip(fn () => $dbForConsole->find('repositories', [
|
||||
Query::equal('providerRepositoryId', [$providerRepositoryId]),
|
||||
Query::orderDesc('$createdAt')
|
||||
]);
|
||||
]));
|
||||
|
||||
$createGitDeployments($github, $providerInstallationId, $repositories, $providerBranch, $providerBranchUrl, $providerRepositoryName, $providerRepositoryUrl, $providerRepositoryOwner, $providerCommitHash, $providerCommitAuthor, $providerCommitAuthorUrl, $providerCommitMessage, $providerCommitUrl, $providerPullRequestId, $external, $dbForConsole, $queueForBuilds, $getProjectDB, $request);
|
||||
} elseif ($parsedPayload["action"] == "closed") {
|
||||
|
@ -929,10 +929,10 @@ App::post('/v1/vcs/github/events')
|
|||
$external = $parsedPayload["external"] ?? true;
|
||||
|
||||
if ($external) {
|
||||
$repositories = $dbForConsole->find('repositories', [
|
||||
$repositories = Authorization::skip(fn () => $dbForConsole->find('repositories', [
|
||||
Query::equal('providerRepositoryId', [$providerRepositoryId]),
|
||||
Query::orderDesc('$createdAt')
|
||||
]);
|
||||
]));
|
||||
|
||||
foreach ($repositories as $repository) {
|
||||
$providerPullRequestIds = $repository->getAttribute('providerPullRequestIds', []);
|
||||
|
@ -1092,9 +1092,9 @@ App::patch('/v1/vcs/github/installations/:installationId/repositories/:repositor
|
|||
throw new Exception(Exception::INSTALLATION_NOT_FOUND);
|
||||
}
|
||||
|
||||
$repository = $dbForConsole->getDocument('repositories', $repositoryId, [
|
||||
$repository = Authorization::skip(fn () => $dbForConsole->getDocument('repositories', $repositoryId, [
|
||||
Query::equal('projectInternalId', [$project->getInternalId()])
|
||||
]);
|
||||
]));
|
||||
|
||||
if ($repository->isEmpty()) {
|
||||
throw new Exception(Exception::REPOSITORY_NOT_FOUND);
|
||||
|
@ -1109,7 +1109,7 @@ App::patch('/v1/vcs/github/installations/:installationId/repositories/:repositor
|
|||
|
||||
// TODO: Delete from array when PR is closed
|
||||
|
||||
$repository = $dbForConsole->updateDocument('repositories', $repository->getId(), $repository);
|
||||
$repository = Authorization::skip(fn () => $dbForConsole->updateDocument('repositories', $repository->getId(), $repository));
|
||||
|
||||
$privateKey = App::getEnv('_APP_VCS_GITHUB_PRIVATE_KEY');
|
||||
$githubAppId = App::getEnv('_APP_VCS_GITHUB_APP_ID');
|
||||
|
|
3
bin/patch-recreate-repositories-documents
Normal file
3
bin/patch-recreate-repositories-documents
Normal file
|
@ -0,0 +1,3 @@
|
|||
#!/bin/sh
|
||||
|
||||
php /usr/src/code/app/cli.php patch-recreate-repositories-documents $@
|
|
@ -19,6 +19,7 @@ use Appwrite\Platform\Tasks\VolumeSync;
|
|||
use Appwrite\Platform\Tasks\CalcTierStats;
|
||||
use Appwrite\Platform\Tasks\Upgrade;
|
||||
use Appwrite\Platform\Tasks\DeleteOrphanedProjects;
|
||||
use Appwrite\Platform\Tasks\PatchRecreateRepositoriesDocuments;
|
||||
|
||||
class Tasks extends Service
|
||||
{
|
||||
|
@ -42,6 +43,7 @@ class Tasks extends Service
|
|||
->addAction(Specs::getName(), new Specs())
|
||||
->addAction(CalcTierStats::getName(), new CalcTierStats())
|
||||
->addAction(DeleteOrphanedProjects::getName(), new DeleteOrphanedProjects())
|
||||
->addAction(PatchRecreateRepositoriesDocuments::getName(), new PatchRecreateRepositoriesDocuments())
|
||||
|
||||
;
|
||||
}
|
||||
|
|
|
@ -0,0 +1,151 @@
|
|||
<?php
|
||||
|
||||
namespace Appwrite\Platform\Tasks;
|
||||
|
||||
use Utopia\Platform\Action;
|
||||
use Utopia\CLI\Console;
|
||||
use Utopia\Database\Database;
|
||||
use Utopia\Database\Document;
|
||||
use Utopia\Database\Helpers\ID;
|
||||
use Utopia\Database\Helpers\Permission;
|
||||
use Utopia\Database\Helpers\Role;
|
||||
use Utopia\Database\Query;
|
||||
use Utopia\Validator\Text;
|
||||
|
||||
class PatchRecreateRepositoriesDocuments extends Action
|
||||
{
|
||||
public static function getName(): string
|
||||
{
|
||||
return 'patch-recreate-repositories-documents';
|
||||
}
|
||||
|
||||
public function __construct()
|
||||
{
|
||||
$this
|
||||
->desc('Recreate missing repositories in consoleDB from projectDBs. They can be missing if you used Appwrite 1.4.10 or 1.4.11, and deleted a function.')
|
||||
->param('after', '', new Text(36), 'After cursor', true)
|
||||
->param('projectId', '', new Text(36), 'Select project to validate', true)
|
||||
->inject('dbForConsole')
|
||||
->inject('getProjectDB')
|
||||
->callback(fn ($after, $projectId, $dbForConsole, $getProjectDB) => $this->action($after, $projectId, $dbForConsole, $getProjectDB));
|
||||
}
|
||||
|
||||
public function action($after, $projectId, Database $dbForConsole, callable $getProjectDB): void
|
||||
{
|
||||
Console::info("Starting the patch");
|
||||
|
||||
$startTime = microtime(true);
|
||||
|
||||
if (!empty($projectId)) {
|
||||
$project = $dbForConsole->getDocument('projects', $projectId);
|
||||
$dbForProject = call_user_func($getProjectDB, $project);
|
||||
$this->recreateRepositories($dbForConsole, $dbForProject, $project);
|
||||
} else {
|
||||
$queries = [];
|
||||
if (!empty($after)) {
|
||||
Console::info("Iterating remaining projects after project with ID {$after}");
|
||||
$project = $dbForConsole->getDocument('projects', $after);
|
||||
$queries = [Query::cursorAfter($project)];
|
||||
} else {
|
||||
Console::info("Iterating all projects");
|
||||
}
|
||||
$this->foreachDocument($dbForConsole, 'projects', $queries, function (Document $project) use ($getProjectDB, $dbForConsole) {
|
||||
$dbForProject = call_user_func($getProjectDB, $project);
|
||||
$this->recreateRepositories($dbForConsole, $dbForProject, $project);
|
||||
});
|
||||
}
|
||||
|
||||
$endTime = microtime(true);
|
||||
$timeTaken = $endTime - $startTime;
|
||||
|
||||
$hours = (int)($timeTaken / 3600);
|
||||
$timeTaken -= $hours * 3600;
|
||||
$minutes = (int)($timeTaken / 60);
|
||||
$timeTaken -= $minutes * 60;
|
||||
$seconds = (int)$timeTaken;
|
||||
$milliseconds = ($timeTaken - $seconds) * 1000;
|
||||
Console::info("Recreate patch completed in $hours h, $minutes m, $seconds s, $milliseconds mis ( total $timeTaken milliseconds)");
|
||||
}
|
||||
|
||||
protected function foreachDocument(Database $database, string $collection, array $queries = [], callable $callback = null): void
|
||||
{
|
||||
$limit = 1000;
|
||||
$results = [];
|
||||
$sum = $limit;
|
||||
$latestDocument = null;
|
||||
|
||||
while ($sum === $limit) {
|
||||
$newQueries = $queries;
|
||||
|
||||
if ($latestDocument != null) {
|
||||
array_unshift($newQueries, Query::cursorAfter($latestDocument));
|
||||
}
|
||||
$newQueries[] = Query::limit($limit);
|
||||
$results = $database->find($collection, $newQueries);
|
||||
|
||||
if (empty($results)) {
|
||||
return;
|
||||
}
|
||||
|
||||
$sum = count($results);
|
||||
|
||||
foreach ($results as $document) {
|
||||
if (is_callable($callback)) {
|
||||
$callback($document);
|
||||
}
|
||||
}
|
||||
$latestDocument = $results[array_key_last($results)];
|
||||
}
|
||||
}
|
||||
|
||||
public function recreateRepositories(Database $dbForConsole, Database $dbForProject, Document $project): void
|
||||
{
|
||||
$projectId = $project->getId();
|
||||
Console::log("Running patch for project {$projectId}");
|
||||
|
||||
$this->foreachDocument($dbForProject, 'functions', [], function (Document $function) use ($dbForProject, $dbForConsole, $project) {
|
||||
$isConnected = !empty($function->getAttribute('providerRepositoryId', ''));
|
||||
|
||||
if ($isConnected) {
|
||||
$repository = $dbForConsole->getDocument('repositories', $function->getAttribute('repositoryId', ''));
|
||||
|
||||
if ($repository->isEmpty()) {
|
||||
$projectId = $project->getId();
|
||||
$functionId = $function->getId();
|
||||
Console::success("Recreating repositories document for project ID {$projectId}, function ID {$functionId}");
|
||||
|
||||
$repository = $dbForConsole->createDocument('repositories', new Document([
|
||||
'$id' => ID::unique(),
|
||||
'$permissions' => [
|
||||
Permission::read(Role::any()),
|
||||
Permission::update(Role::any()),
|
||||
Permission::delete(Role::any()),
|
||||
],
|
||||
'installationId' => $function->getAttribute('installationId', ''),
|
||||
'installationInternalId' => $function->getAttribute('installationInternalId', ''),
|
||||
'projectId' => $project->getId(),
|
||||
'projectInternalId' => $project->getInternalId(),
|
||||
'providerRepositoryId' => $function->getAttribute('providerRepositoryId', ''),
|
||||
'resourceId' => $function->getId(),
|
||||
'resourceInternalId' => $function->getInternalId(),
|
||||
'resourceType' => 'function',
|
||||
'providerPullRequestIds' => []
|
||||
]));
|
||||
|
||||
$function = $dbForProject->updateDocument('functions', $function->getId(), $function
|
||||
->setAttribute('repositoryId', $repository->getId())
|
||||
->setAttribute('repositoryInternalId', $repository->getInternalId()));
|
||||
|
||||
$this->foreachDocument($dbForProject, 'deployments', [
|
||||
Query::equal('resourceInternalId', [$function->getInternalId()]),
|
||||
Query::equal('resourceType', ['functions'])
|
||||
], function (Document $deployment) use ($dbForProject, $repository) {
|
||||
$dbForProject->updateDocument('deployments', $deployment->getId(), $deployment
|
||||
->setAttribute('repositoryId', $repository->getId())
|
||||
->setAttribute('repositoryInternalId', $repository->getInternalId()));
|
||||
});
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
|
@ -735,10 +735,10 @@ class Deletes extends Action
|
|||
Query::equal('resourceType', ['function']),
|
||||
], $dbForConsole, function (Document $document) use ($dbForConsole) {
|
||||
$providerRepositoryId = $document->getAttribute('providerRepositoryId', '');
|
||||
$projectId = $document->getAttribute('projectId', '');
|
||||
$projectInternalId = $document->getAttribute('projectInternalId', '');
|
||||
$this->deleteByGroup('vcsComments', [
|
||||
Query::equal('providerRepositoryId', [$providerRepositoryId]),
|
||||
Query::equal('projectId', [$projectId]),
|
||||
Query::equal('projectInternalId', [$projectInternalId]),
|
||||
], $dbForConsole);
|
||||
});
|
||||
|
||||
|
|
Loading…
Reference in a new issue