VCS integration edge cases fix

2024-06-30 04:00:34 +12:00 · 2023-06-17 12:07:30 +02:00 · 2023-06-17 12:07:30 +02:00 · 5237e24739
parent 41346b017a
commit 5237e24739
7 changed files with 127 additions and 26 deletions
--- a/.env
+++ b/.env
@ -13,10 +13,10 @@ _APP_SYSTEM_RESPONSE_FORMAT=
 _APP_OPTIONS_ABUSE=disabled
 _APP_OPTIONS_FORCE_HTTPS=disabled
 _APP_OPENSSL_KEY_V1=your-secret-key
-_APP_DOMAIN=localhost
+_APP_DOMAIN=matejbaco.loca.lt
 _APP_DOMAIN_FUNCTIONS=functions.localhost
 _APP_DOMAIN_DO_TOKEN=
-_APP_DOMAIN_TARGET=localhost
+_APP_DOMAIN_TARGET=matejbaco.loca.lt
 _APP_REDIS_HOST=redis
 _APP_REDIS_PORT=6379
 _APP_REDIS_PASS=
@ -81,7 +81,33 @@ _APP_DOCKER_HUB_USERNAME=
 _APP_DOCKER_HUB_PASSWORD=
 _APP_CONSOLE_GITHUB_SECRET=
 _APP_CONSOLE_GITHUB_APP_ID=
-VCS_GITHUB_APP_NAME=
-VCS_GITHUB_PRIVATE_KEY=
-VCS_GITHUB_APP_ID=
-VCS_GITHUB_WEBHOOK_SECRET=
+VCS_GITHUB_APP_NAME=appwrite-generated-on-22-5-2023
+VCS_GITHUB_PRIVATE_KEY="-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEAoFvSnF1pmTlSwS5QUXvLqhXxaU9AZ9LASIw/qJNpTTodcbeg
+i7Fj5POG8wDexbzAajpPfwxFoRqQ+A69azB2cskgh2sRzB6q3KdjNQIw/+5JQmQK
+EF46s6LFPq7xjZCw1j8buQcHtmXJQD3D6K0hPst9N7xOjMYHY0wCR9hAfx0BCd9l
+B8s+OnhKgZDWegq2I8S2ukTtVv9MxYC4Di9pj66QmJDXoBkLllD88DwZDBQh2spt
+KFxPOe5Ig7rhqZu9bPrCtQQBwqFnn3bURbCZA99eyv4z2eiYO6WmOLF6705c1QC7
+t+WDbTjMHBBJJZeM1Hzcd8035rMvWxkHtRTjZQIDAQABAoIBAAoKFaJyerT8EMFF
+bAPd8z8/xnBoZMjbOeyQ0z9zkykhlUxUSbnVLkLcODuod0n76jP8WJPEqetPJXpS
+OeunChNfoii4PFICLYJI/H4KGBU9ztpYNwUVUSmD7ENjYn4Ra/DN3Kd12DbEIphi
+BYzgoe8zgDBoGqbXkwjVbmMfF9KdXdteIvy3Qa8CdrJxgGJ7O/RaD9MzBaumpP9e
+61fetvXpceOCVY3ENwAj3ptNjfBY5dAknMb9N6F4f6bT5Bx0YaqUWQ3LTmpFuNFR
+4UOOgF8BUHUGomIioKZroReQUSn8T2ISGxpI7N02o1uXZvA8FjdRo0ZMFl5iMJ8A
+HhP+DlUCgYEA0g8fh8WuGnz5AMQXGVPNzLO4qd+TgR34+o/Dd029FIZeEFTWJMvM
+ytpNaTOU6y0F9fyjp8UuCyJFT6MN6svQ4qIhVBVeR8vAggquNeahyaik3QtWukVX
+h9rXqdKeJGCUrFlmRfmKm5nqy17wox5fL4p1fM6F5T19IWx/h1hy3UsCgYEAw24N
+hZrSbgi8A92IiW0j/V3ztK0X0YHxYP8GhUXXC69hXBRL4k3qvx8KVNVpxN6RABCp
+3U8XJEqQoZGm2oSp9z8GYNNpUQuFrA1IvPqueqgt3+EHaQ97HWpVWPp73S77/ZPI
+DwULiCrvoONiwDGVbVyBoK3fhWt4IPM7qCl2RA8CgYAROyxa9txRT+8dIPjEGxvG
+rvd29+Aht1O1OhXOs5HPHTaaIFtwqUcjXMT83D5bDFZw10iVepdMUoNdBQMfP5ji
+/mm9fAM/4HbqM/YWddTPtgXQHB4BKa8Ede86/wv57/JCMRMyqkW0fevEwwT6XINy
+YulAU7CoHWMmWsAbWiYSAQKBgBwrtglHSwQPgEKyYWcl82UncOTfeqp5p7jKqj71
+1Uza6Bbk2hdyEd+g8tVNuiRlzNMMk9/R8c0KbpTLsRiYvQNl815ruyeymH83nv+W
+xyXpLkuyXWYYlC+mT003Njmo9ywTbGg12vay8b3oaSi5PQf5qLnntmu+FR/yz+uq
+pmCjAoGBAKU7FqRFVo2zyLvaeTdzNB6XWPhNl2dbazhPtsYAyHBB7LuU+rV4un0t
+56WhIhwYMbOW4Mgkdj7DrhlWY/b9BugrkScv/WevNo2fE+QWQK9klJCfmdJqo9tW
+58doaZSBqlYo2cAuaOXchexRyMDkFRBhRC4A7X0jAFoalSW5NyeV
+-----END RSA PRIVATE KEY-----"
+VCS_GITHUB_APP_ID=337303
+VCS_GITHUB_WEBHOOK_SECRET=c41d6d6029a170a2cd5970e63cdb2a802a03cfbcno
--- a/app/config/collections.php
+++ b/app/config/collections.php
@ -2901,7 +2901,7 @@ $collections = [
                'array' => false,
            ],
            [
-                '$id' => ID::custom('vcsRepositoryInternalId'),
+                '$id' => ID::custom('vcsRepositoryDocId'),
                'type' => Database::VAR_STRING,
                'signed' => true,
                'size' => Database::LENGTH_KEY,
@ -2910,6 +2910,17 @@ $collections = [
                'required' => false,
                'array' => false,
            ],
+            [
+                '$id' => ID::custom('vcsRepositoryDocInternalId'),
+                'type' => Database::VAR_STRING,
+                'format' => '',
+                'size' => Database::LENGTH_KEY,
+                'signed' => true,
+                'required' => false,
+                'default' => null,
+                'array' => false,
+                'filters' => [],
+            ],
            [
                '$id' => ID::custom('vcsBranch'),
                'type' => Database::VAR_STRING,
--- a/app/controllers/api/functions.php
+++ b/app/controllers/api/functions.php
@ -65,8 +65,9 @@ $redeployVcsLogic = function (Document $function, Document $project, Document $i
        'type' => 'vcs',
        'vcsInstallationId' => $installation->getId(),
        'vcsInstallationInternalId' => $installation->getInternalId(),
-        'vcsRepositoryId' => $function->getAttribute('vcsRepositoryDocId', ''),
-        'vcsRepositoryInternalId' => $function->getAttribute('vcsRepositoryDocInternalId', ''),
+        'vcsRepositoryId' => $function->getAttribute('vcsRepositoryId', ''),
+        'vcsRepositoryDocId' => $function->getAttribute('vcsRepositoryDocId', ''),
+        'vcsRepositoryDocInternalId' => $function->getAttribute('vcsRepositoryDocInternalId', ''),
        'vcsBranch' => $function->getAttribute('vcsBranch', 'main'),
        'vcsRootDirectory' => $function->getAttribute('vcsRootDirectory', ''),
        'search' => implode(' ', [$deploymentId, $entrypoint]),
--- a/app/controllers/api/vcs.php
+++ b/app/controllers/api/vcs.php
@ -320,7 +320,7 @@ App::get('/v1/vcs/github/installations/:installationId/repositories/:repositoryI
        ]), Response::MODEL_BRANCH_LIST);
    });

-$createGitDeployments = function (GitHub $github, string $installationId, array $vcsRepos, string $branchName, string $SHA, string $commentId, Database $dbForConsole, callable $getProjectDB, Request $request) {
+$createGitDeployments = function (GitHub $github, string $installationId, string $repositoryId, array $vcsRepos, string $branchName, string $SHA, string $commentId, Database $dbForConsole, callable $getProjectDB, Request $request) {
    foreach ($vcsRepos as $resource) {
        $resourceType = $resource->getAttribute('resourceType');

@ -345,7 +345,7 @@ $createGitDeployments = function (GitHub $github, string $installationId, array
            }

            $latestDeployment = Authorization::skip(fn () => $dbForProject->findOne('deployments', [
-                Query::equal('vcsRepositoryId', [$vcsRepoId]),
+                Query::equal('vcsRepositoryId', [$repositoryId]),
                Query::equal('vcsBranch', [$branchName]),
                Query::equal('resourceType', ['functions']),
                Query::orderDesc('$createdAt'),
@ -367,6 +367,23 @@ $createGitDeployments = function (GitHub $github, string $installationId, array
                }
            }

+            $owner = $github->getOwnerName($installationId);
+            $repositoryName = $github->getRepositoryName($repositoryId);
+
+            $comment = new Comment();
+            // TODO: Add all builds reeeeee
+            $comment->addBuild($project, $function, 'waiting', $deploymentId);
+
+            if (empty($latestCommentId)) {
+                $pullRequest = $github->getBranchPullRequest($owner, $repositoryName, $branchName);
+                if (!empty($pullRequest)) {
+                    $pullRequestNumber = \strval($pullRequest['number']);
+                    $latestCommentId = $github->createComment($owner, $repositoryName, $pullRequestNumber, $comment->generateComment());
+                }
+            } else {
+                $latestCommentId = $github->updateComment($owner, $repositoryName, $latestCommentId, $comment->generateComment());
+            }
+
            $deployment = $dbForProject->createDocument('deployments', new Document([
                '$id' => $deploymentId,
                '$permissions' => [
@ -382,9 +399,10 @@ $createGitDeployments = function (GitHub $github, string $installationId, array
                'type' => 'vcs',
                'vcsInstallationId' => $vcsInstallationId,
                'vcsInstallationInternalId' => $vcsInstallationInternalId,
-                'vcsRepositoryId' => $vcsRepoId,
-                'vcsRepositoryInternalId' => $vcsRepoInternalId,
-                'vcsCommentId' => $latestCommentId,
+                'vcsRepositoryId' => $repositoryId,
+                'vcsRepositoryDocId' => $vcsRepoId,
+                'vcsRepositoryDocInternalId' => $vcsRepoInternalId,
+                'vcsCommentId' => \strval($latestCommentId),
                'vcsBranch' => $branchName,
                'search' => implode(' ', [$deploymentId, $function->getAttribute('entrypoint')]),
                'activate' => $activate,
@ -461,7 +479,7 @@ App::post('/v1/vcs/github/incomingwebhook')
                    Query::limit(100),
                ]);

-                $createGitDeployments($github, $installationId, $vcsRepos, $branchName, $SHA, '', $dbForConsole, $getProjectDB, $request);
+                $createGitDeployments($github, $installationId, $repositoryId, $vcsRepos, $branchName, $SHA, '', $dbForConsole, $getProjectDB, $request);
            } elseif ($event == $github::EVENT_INSTALLATION) {
                if ($parsedPayload["action"] == "deleted") {
                    // TODO: Use worker for this job instead (update function as well)
@ -512,7 +530,7 @@ App::post('/v1/vcs/github/incomingwebhook')
                            $vcsRepoId = $vcsRepo->getId();

                            $deployment = Authorization::skip(fn () => $dbForProject->findOne('deployments', [
-                                Query::equal('vcsRepositoryId', [$vcsRepoId]),
+                                Query::equal('vcsRepositoryDocId', [$vcsRepoId]),
                                Query::equal('vcsBranch', [$branchName]),
                                Query::equal('resourceType', ['functions']),
                                Query::orderDesc('$createdAt'),
@ -520,7 +538,7 @@ App::post('/v1/vcs/github/incomingwebhook')

                            if (!$deployment || $deployment->isEmpty()) {
                                $function = Authorization::skip(fn () => $dbForProject->findOne('functions', [
-                                    Query::equal('vcsRepositoryId', [$vcsRepoId]),
+                                    Query::equal('vcsRepositoryDocId', [$vcsRepoId]),
                                    Query::orderDesc('$createdAt'),
                                ]));
                                $build = new Document([]);
@ -544,7 +562,7 @@ App::post('/v1/vcs/github/incomingwebhook')
                            $commentId = $github->createComment($owner, $repositoryName, $pullRequestNumber, $comment->generateComment());
                        }

-                        $createGitDeployments($github, $installationId, $vcsRepos, $branchName, '', $commentId, $dbForConsole, $getProjectDB, $request);
+                        $createGitDeployments($github, $installationId, $repositoryId, $vcsRepos, $branchName, '', $commentId, $dbForConsole, $getProjectDB, $request);
                    }
                }
            }
--- a/app/workers/builds.php
+++ b/app/workers/builds.php
@ -9,7 +9,6 @@ use Appwrite\Utopia\Response\Model\Deployment;
 use Executor\Executor;
 use Appwrite\Usage\Stats;
 use Appwrite\Vcs\Comment;
-use Swoole\Runtime;
 use Utopia\Database\DateTime;
 use Utopia\App;
 use Utopia\CLI\Console;
@ -106,12 +105,11 @@ class BuildsV1 extends Worker
            $buildId = ID::unique();

            $vcsInstallationId = $deployment->getAttribute('vcsInstallationId', '');
-            $vcsRepoId = $deployment->getAttribute('vcsRepositoryId', '');
-            $isVcsEnabled = $vcsRepoId ? true : false;
+            $repositoryId = $deployment->getAttribute('vcsRepositoryId', '');
+            $isVcsEnabled = $repositoryId ? true : false;

            if ($isVcsEnabled) {
-                $vcsRepos = Authorization::skip(fn () => $dbForConsole->getDocument('vcsRepos', $vcsRepoId));
-                $repositoryId = $vcsRepos->getAttribute('repositoryId');
+                \var_dump("Will clone");
                $vcsInstallations = Authorization::skip(fn () => $dbForConsole->getDocument('vcsInstallations', $vcsInstallationId));
                $installationId = $vcsInstallations->getAttribute('installationId');

@ -129,6 +127,7 @@ class BuildsV1 extends Worker
                $repositoryName = $github->getRepositoryName($repositoryId);
                $branchName = $deployment->getAttribute('vcsBranch');
                $gitCloneCommand = $github->generateGitCloneCommand($owner, $repositoryId, $branchName, $tmpDirectory, $rootDirectory);
+                \var_dump($gitCloneCommand);
                $stdout = '';
                $stderr = '';
                Console::execute('mkdir -p /tmp/builds/' . $buildId, '', $stdout, $stderr);
@ -251,6 +250,7 @@ class BuildsV1 extends Worker
        $source = $deployment->getAttribute('path');

        if ($isVcsEnabled) {
+            \var_dump("Changing source");
            $source = $path;
        }

@ -275,6 +275,7 @@ class BuildsV1 extends Worker

            $response = null;

+            // TODO: Remove run() wrapper when switching to new utopia queue. That should be done on Swoole adapter in the libary
            Co\run(function () use ($project, $deployment, &$response, $source, $function, $runtime, $vars, $command, &$build, $dbForProject, $allEvents) {
                Co::join([
                    Co\go(function () use ($project, $deployment, &$response, &$build, $dbForProject, $allEvents) {
@ -444,10 +445,11 @@ class BuildsV1 extends Worker
            $github->updateCommitStatus($repositoryName, $SHA, $owner, $state, $message, $targetUrl, $name);
        }

-
+        // TODO: Fix race condition
        if (!empty($commentId)) {
            $comment = new Comment();
            $comment->parseComment($github->getComment($owner, $repositoryName, $commentId));
+            \sleep(5);
            $comment->addBuild($project, $function, $status, $deployment->getId());
            $github->updateComment($owner, $repositoryName, $commentId, $comment->generateComment());
        }
--- a/composer.lock
+++ b/composer.lock
@ -2705,7 +2705,7 @@
            "source": {
                "type": "git",
                "url": "https://github.com/utopia-php/vcs.git",
-                "reference": "01afc324865613f7b34ca9981745bfaaec8ec363"
+                "reference": "2966e482920a9dd028a237083a81de723a9bcbf2"
            },
            "require": {
                "adhocore/jwt": "^1.1",
@ -2750,7 +2750,7 @@
                "utopia",
                "vcs"
            ],
-            "time": "2023-06-15T10:13:29+00:00"
+            "time": "2023-06-16T11:02:30+00:00"
        },
        {
            "name": "utopia-php/websocket",
--- a/docker-compose.yml
+++ b/docker-compose.yml
@ -422,6 +422,49 @@ services:
      - VCS_GITHUB_PRIVATE_KEY
      - VCS_GITHUB_APP_ID

+  appwrite-worker-builds2:
+    entrypoint: worker-builds
+    <<: *x-logging
+    container_name: appwrite-worker-builds2
+    image: appwrite-dev
+    networks:
+      - appwrite
+    volumes:
+      - appwrite-functions:/storage/functions:rw
+      - appwrite-builds:/storage/builds:rw
+      - ./app:/usr/src/code/app
+      - ./src:/usr/src/code/src
+    depends_on:
+      - redis
+      - mariadb
+    environment:
+      - _APP_ENV
+      - _APP_WORKER_PER_CORE
+      - _APP_CONNECTIONS_MAX
+      - _APP_POOL_CLIENTS
+      - _APP_OPENSSL_KEY_V1
+      - _APP_EXECUTOR_SECRET
+      - _APP_EXECUTOR_HOST
+      - _APP_DB_HOST
+      - _APP_DB_PORT
+      - _APP_DB_SCHEMA
+      - _APP_DB_USER
+      - _APP_DB_PASS
+      - _APP_REDIS_HOST
+      - _APP_REDIS_PORT
+      - _APP_REDIS_USER
+      - _APP_REDIS_PASS
+      - _APP_CONNECTIONS_DB_CONSOLE
+      - _APP_CONNECTIONS_DB_PROJECT
+      - _APP_CONNECTIONS_CACHE
+      - _APP_CONNECTIONS_QUEUE
+      - _APP_CONNECTIONS_STORAGE
+      - _APP_LOGGING_PROVIDER
+      - _APP_LOGGING_CONFIG
+      - VCS_GITHUB_APP_NAME
+      - VCS_GITHUB_PRIVATE_KEY
+      - VCS_GITHUB_APP_ID
+
  appwrite-worker-certificates:
    entrypoint: worker-certificates
    <<: *x-logging