Merge branch '1.5.x' of github.com:appwrite/appwrite into db-project-by-region

# Conflicts:
#	app/controllers/general.php
#	src/Appwrite/Platform/Tasks/Migrate.php

.gitmodules

@@ -1,4 +1,4 @@
 [submodule "app/console"]
 	path = app/console
 	url = https://github.com/appwrite/console
-	branch = 4.3.23
+	branch = 4.3.27

app/console

@@ -1 +1 @@
-Subproject commit 112fccf5a044c9795b31b264f5974224a3545a17
+Subproject commit e0828010f4c222bb99cf65ddfed288e60602666f

app/controllers/api/teams.php

@@ -669,6 +669,7 @@ App::post('/v1/teams/:teamId/memberships')
         }
 
         $queueForEvents
+            ->setParam('userId', $invitee->getId())
             ->setParam('teamId', $team->getId())
             ->setParam('membershipId', $membership->getId())
         ;
@@ -902,6 +903,7 @@ App::patch('/v1/teams/:teamId/memberships/:membershipId')
         $dbForProject->purgeCachedDocument('users', $profile->getId());
 
         $queueForEvents
+            ->setParam('userId', $profile->getId())
             ->setParam('teamId', $team->getId())
             ->setParam('membershipId', $membership->getId());
 
@@ -1027,6 +1029,7 @@ App::patch('/v1/teams/:teamId/memberships/:membershipId/status')
         Authorization::skip(fn () => $dbForProject->increaseDocumentAttribute('teams', $team->getId(), 'total', 1));
 
         $queueForEvents
+            ->setParam('userId', $user->getId())
             ->setParam('teamId', $team->getId())
             ->setParam('membershipId', $membership->getId())
         ;
@@ -1108,6 +1111,7 @@ App::delete('/v1/teams/:teamId/memberships/:membershipId')
         }
 
         $queueForEvents
+            ->setParam('userId', $user->getId())
             ->setParam('teamId', $team->getId())
             ->setParam('membershipId', $membership->getId())
             ->setPayload($response->output($membership, Response::MODEL_MEMBERSHIP))

app/controllers/general.php

@@ -49,7 +49,7 @@ function router(App $utopia, Database $dbForConsole, callable $getProjectDB, Swo
     $route = Authorization::skip(
         fn () => $dbForConsole->find('rules', [
             Query::equal('domain', [$host]),
-            Query::limit(1),
+            Query::limit(1)
         ])
     )[0] ?? null;
 
@@ -70,7 +70,6 @@ function router(App $utopia, Database $dbForConsole, callable $getProjectDB, Swo
 
         // Act as API - no Proxy logic
         $utopia->getRoute()?->label('error', '');
-
         return false;
     }
 
@@ -113,6 +112,7 @@ function router(App $utopia, Database $dbForConsole, callable $getProjectDB, Swo
             $path .= '?' . $query;
         }
 
+
         $body = $swooleRequest->getContent() ?? '';
         $method = $swooleRequest->server['request_method'];
 
@@ -228,7 +228,7 @@ function router(App $utopia, Database $dbForConsole, callable $getProjectDB, Swo
                 'APPWRITE_FUNCTION_TRIGGER' => $headers['x-appwrite-trigger'] ?? '',
                 'APPWRITE_FUNCTION_DATA' => $body ?? '',
                 'APPWRITE_FUNCTION_USER_ID' => $headers['x-appwrite-user-id'] ?? '',
-                'APPWRITE_FUNCTION_JWT' => $headers['x-appwrite-user-jwt'] ?? '',
+                'APPWRITE_FUNCTION_JWT' => $headers['x-appwrite-user-jwt'] ?? ''
             ]);
         }
 
@@ -311,7 +311,10 @@ function router(App $utopia, Database $dbForConsole, callable $getProjectDB, Swo
                 ->addMetric(METRIC_EXECUTIONS_COMPUTE, (int)($execution->getAttribute('duration') * 1000)) // per project
                 ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE), (int)($execution->getAttribute('duration') * 1000)) // per function
                 ->addMetric(METRIC_EXECUTIONS_MB_SECONDS, (int)(512 * $execution->getAttribute('duration', 0)))
-                ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_MB_SECONDS), (int) (512 * $execution->getAttribute('duration', 0)));
+                ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_MB_SECONDS), (int)(512 * $execution->getAttribute('duration', 0)))
+                ->setProject($project)
+                ->trigger()
+            ;
 
             if ($function->getAttribute('logging')) {
                 /** @var Document $execution */
@@ -356,14 +359,12 @@ function router(App $utopia, Database $dbForConsole, callable $getProjectDB, Swo
         return true;
     } elseif ($type === 'api') {
         $utopia->getRoute()?->label('error', '');
-
         return false;
     } else {
         throw new AppwriteException(AppwriteException::GENERAL_SERVER_ERROR, 'Unknown resource type ' . $type);
     }
 
     $utopia->getRoute()?->label('error', '');
-
     return false;
 }
 
@@ -470,7 +471,7 @@ App::init()
                     Console::warning($domain->get() . ' is not a main domain. Skipping SSL certificate generation.');
                 } else {
                     $domainDocument = $dbForConsole->findOne('rules', [
-                        Query::equal('domain', [$domain->get()]),
+                        Query::equal('domain', [$domain->get()])
                     ]);
 
                     if (!$domainDocument) {
@@ -479,7 +480,7 @@ App::init()
                             'resourceType' => 'api',
                             'status' => 'verifying',
                             'projectId' => 'console',
-                            'projectInternalId' => 'console',
+                            'projectInternalId' => 'console'
                         ]);
 
                         $domainDocument = $dbForConsole->createDocument('rules', $domainDocument);
@@ -559,7 +560,7 @@ App::init()
                 $response->addFilter(new ResponseV17());
             }
             if (version_compare($responseFormat, APP_VERSION_STABLE, '>')) {
-                $response->addHeader('X-Appwrite-Warning', 'The current SDK is built for Appwrite '.$responseFormat.'. However, the current Appwrite server version is '.APP_VERSION_STABLE.'. Please downgrade your SDK to match the Appwrite version: https://appwrite.io/docs/sdks');
+                $response->addHeader('X-Appwrite-Warning', "The current SDK is built for Appwrite " . $responseFormat . ". However, the current Appwrite server version is ". APP_VERSION_STABLE . ". Please downgrade your SDK to match the Appwrite version: https://appwrite.io/docs/sdks");
             }
         }
 
@@ -732,7 +733,7 @@ App::error()
 
             if (!(empty($providerName) || empty($providerConfig))) {
                 if (!Logger::hasProvider($providerName)) {
-                    throw new Exception('Logging provider not supported. Logging is disabled');
+                    throw new Exception("Logging provider not supported. Logging is disabled");
                 }
 
                 $classname = '\\Utopia\\Logger\\Adapter\\' . \ucfirst($providerName);
@@ -762,6 +763,7 @@ App::error()
                 ->trigger();
         }
 
+
         if ($logger && $publish) {
             try {
                 /** @var Utopia\Database\Document $user */
@@ -781,7 +783,7 @@ App::error()
                 $dsn = new DSN('mysql://' . $project->getAttribute('database', 'console'));
             }
 
-            $log->setNamespace('http');
+            $log->setNamespace("http");
             $log->setServer(\gethostname());
             $log->setVersion($version);
             $log->setType(Log::TYPE_ERROR);
@@ -801,7 +803,7 @@ App::error()
             $log->addExtra('trace', $error->getTraceAsString());
             $log->addExtra('roles', Authorization::getRoles());
 
-            $action = $route->getLabel('sdk.namespace', 'UNKNOWN_NAMESPACE').'.'.$route->getLabel('sdk.method', 'UNKNOWN_METHOD');
+            $action = $route->getLabel("sdk.namespace", "UNKNOWN_NAMESPACE") . '.' . $route->getLabel("sdk.method", "UNKNOWN_METHOD");
             $log->setAction($action);
 
             $isProduction = System::getEnv('_APP_ENV', 'development') === 'production';
@@ -950,7 +952,7 @@ App::get('/.well-known/acme-challenge/*')
             ...Text::ALPHABET_LOWER,
             ...Text::ALPHABET_UPPER,
             '-',
-            '_',
+            '_'
         ]);
 
         if (!$validator->isValid($token) || \count($uriChunks) !== 4) {

app/init.php

@@ -110,7 +110,7 @@ const APP_LIMIT_LIST_DEFAULT = 25; // Default maximum number of items to return
 const APP_KEY_ACCCESS = 24 * 60 * 60; // 24 hours
 const APP_USER_ACCCESS = 24 * 60 * 60; // 24 hours
 const APP_CACHE_UPDATE = 24 * 60 * 60; // 24 hours
-const APP_CACHE_BUSTER = 4323;
+const APP_CACHE_BUSTER = 4327;
 const APP_VERSION_STABLE = '1.5.8';
 const APP_DATABASE_ATTRIBUTE_EMAIL = 'email';
 const APP_DATABASE_ATTRIBUTE_ENUM = 'enum';

app/realtime.php

@@ -380,8 +380,10 @@ $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats,
                         $user = $database->getDocument('users', $userId);
 
                         $roles = Auth::getRoles($user);
+                        $channels = $realtime->connections[$connection]['channels'];
 
-                        $realtime->subscribe($projectId, $connection, $roles, $realtime->connections[$connection]['channels']);
+                        $realtime->unsubscribe($connection);
+                        $realtime->subscribe($projectId, $connection, $roles, $channels);
 
                         $register->get('pools')->reclaim();
                     }

composer.json

@@ -69,7 +69,7 @@
         "utopia-php/storage": "0.18.*",
         "utopia-php/swoole": "0.8.*",
         "utopia-php/system": "0.8.*",
-        "utopia-php/vcs": "0.6.*",
+        "utopia-php/vcs": "0.8.*",
         "utopia-php/websocket": "0.1.*",
         "matomo/device-detector": "6.1.*",
         "dragonmantank/cron-expression": "3.3.2",

composer.lock

@@ -4,7 +4,7 @@
         "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
         "This file is @generated automatically"
     ],
-    "content-hash": "7a9af1063aac524865d2479ce3957d0e",
+    "content-hash": "b5c0db330bf30bd1d240b96116d96baf",
     "packages": [
         {
             "name": "adhocore/jwt",
@@ -2758,16 +2758,16 @@
         },
         {
             "name": "utopia-php/vcs",
-            "version": "0.6.7",
+            "version": "0.8.1",
             "source": {
                 "type": "git",
                 "url": "https://github.com/utopia-php/vcs.git",
-                "reference": "8d8ff1ac68e991b95adb6f91fcde8f9bb8f24974"
+                "reference": "3084aa93d24ed1e70f01e75f4318fc0d07f12596"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/utopia-php/vcs/zipball/8d8ff1ac68e991b95adb6f91fcde8f9bb8f24974",
+                "url": "https://api.github.com/repos/utopia-php/vcs/zipball/3084aa93d24ed1e70f01e75f4318fc0d07f12596",
-                "reference": "8d8ff1ac68e991b95adb6f91fcde8f9bb8f24974",
+                "reference": "3084aa93d24ed1e70f01e75f4318fc0d07f12596",
                 "shasum": ""
             },
             "require": {
@@ -2801,9 +2801,9 @@
             ],
             "support": {
                 "issues": "https://github.com/utopia-php/vcs/issues",
-                "source": "https://github.com/utopia-php/vcs/tree/0.6.7"
+                "source": "https://github.com/utopia-php/vcs/tree/0.8.1"
             },
-            "time": "2024-06-05T17:38:29+00:00"
+            "time": "2024-07-29T20:49:09+00:00"
         },
         {
             "name": "utopia-php/websocket",

src/Appwrite/Platform/Tasks/Migrate.php

@@ -3,8 +3,8 @@
 namespace Appwrite\Platform\Tasks;
 
 use Appwrite\Migration\Migration;
+use Redis;
 use Utopia\App;
-use Utopia\Cache\Cache;
 use Utopia\CLI\Console;
 use Utopia\Database\Database;
 use Utopia\Database\Document;
@@ -12,10 +12,13 @@ use Utopia\Database\Query;
 use Utopia\Database\Validator\Authorization;
 use Utopia\Platform\Action;
 use Utopia\Registry\Registry;
+use Utopia\System\System;
 use Utopia\Validator\Text;
 
 class Migrate extends Action
 {
+    protected Redis $redis;
+
     public static function getName(): string
     {
         return 'migrate';
@@ -27,31 +30,51 @@ class Migrate extends Action
             ->desc('Migrate Appwrite to new version')
             /** @TODO APP_VERSION_STABLE needs to be defined */
             ->param('version', APP_VERSION_STABLE, new Text(8), 'Version to migrate to.', true)
-            ->inject('cache')
             ->inject('dbForConsole')
             ->inject('getProjectDB')
             ->inject('register')
-            ->callback(fn ($version, $cache, $dbForConsole, $getProjectDB, Registry $register) => $this->action($version, $cache, $dbForConsole, $getProjectDB, $register));
+            ->callback(fn ($version, $dbForConsole, $getProjectDB, Registry $register) => $this->action($version, $dbForConsole, $getProjectDB, $register));
+
     }
 
-    private function clearProjectsCache(Cache $cache, Document $project)
+    private function clearProjectsCache(Document $project)
     {
         try {
-            $cache->purge("cache-_{$project->getInternalId()}:*");
+            do {
+                $iterator = null;
+                $pattern = "default-cache-_{$project->getInternalId()}:*";
+                $keys = $this->redis->scan($iterator, $pattern, 1000);
+                if ($keys !== false) {
+                    foreach ($keys as $key) {
+                        $this->redis->del($key);
+                    }
+                }
+            } while ($iterator > 0);
+
         } catch (\Throwable $th) {
             Console::error('Failed to clear project ("'.$project->getId().'") cache with error: '.$th->getMessage());
         }
     }
 
-    public function action(string $version, Cache $cache, Database $dbForConsole, callable $getProjectDB, Registry $register)
+    public function action(string $version, Database $dbForConsole, callable $getProjectDB, Registry $register)
     {
         Authorization::disable();
         if (! array_key_exists($version, Migration::$versions)) {
             Console::error("Version {$version} not found.");
             Console::exit(1);
+
             return;
         }
 
+        $this->redis = new Redis();
+        $this->redis->connect(
+            System::getEnv('_APP_REDIS_HOST', null),
+            System::getEnv('_APP_REDIS_PORT', 6379),
+            3,
+            null,
+            10
+        );
+
         $app = new App('UTC');
 
         Console::success('Starting Data Migration to version '.$version);
@@ -87,7 +110,7 @@ class Migrate extends Action
                     continue;
                 }
 
-                $this->clearProjectsCache($cache, $project);
+                $this->clearProjectsCache($project);
 
                 try {
                     // TODO: Iterate through all project DBs
@@ -103,7 +126,7 @@ class Migrate extends Action
                     throw $th;
                 }
 
-                $this->clearProjectsCache($cache, $project);
+                $this->clearProjectsCache($project);
             }
 
             $sum = \count($projects);

src/Appwrite/Platform/Workers/Builds.php

@@ -217,7 +217,15 @@ class Builds extends Action
 
                 $branchName = $deployment->getAttribute('providerBranch');
                 $commitHash = $deployment->getAttribute('providerCommitHash', '');
-                $gitCloneCommand = $github->generateCloneCommand($cloneOwner, $cloneRepository, $branchName, $tmpDirectory, $rootDirectory, $commitHash);
+
+                $cloneVersion = $branchName;
+                $cloneType = GitHub::CLONE_TYPE_BRANCH;
+                if(!empty($commitHash)) {
+                    $cloneVersion = $commitHash;
+                    $cloneType = GitHub::CLONE_TYPE_COMMIT;
+                }
+
+                $gitCloneCommand = $github->generateCloneCommand($cloneOwner, $cloneRepository, $cloneVersion, $cloneType, $tmpDirectory, $rootDirectory);
                 $stdout = '';
                 $stderr = '';
                 Console::execute('mkdir -p /tmp/builds/' . \escapeshellcmd($buildId), '', $stdout, $stderr);
@@ -240,7 +248,13 @@ class Builds extends Action
                 if (!empty($templateRepositoryName) && !empty($templateOwnerName) && !empty($templateBranch)) {
                     // Clone template repo
                     $tmpTemplateDirectory = '/tmp/builds/' . \escapeshellcmd($buildId) . '/template';
-                    $gitCloneCommandForTemplate = $github->generateCloneCommand($templateOwnerName, $templateRepositoryName, $templateBranch, $tmpTemplateDirectory, $templateRootDirectory);
+
+                    $cloneType = GitHub::CLONE_TYPE_BRANCH;
+                    if(\str_starts_with($templateBranch, '0.1.')) { // Temporary fix for 1.5. In future versions this only support tag names
+                        $cloneType = GitHub::CLONE_TYPE_TAG;
+                    }
+
+                    $gitCloneCommandForTemplate = $github->generateCloneCommand($templateOwnerName, $templateRepositoryName, $templateBranch, $cloneType, $tmpTemplateDirectory, $templateRootDirectory);
                     $exit = Console::execute($gitCloneCommandForTemplate, '', $stdout, $stderr);
 
                     if ($exit !== 0) {

tests/e2e/Services/Functions/FunctionsCustomServerTest.php

@@ -8,6 +8,7 @@ use Tests\E2E\Client;
 use Tests\E2E\Scopes\ProjectCustom;
 use Tests\E2E\Scopes\Scope;
 use Tests\E2E\Scopes\SideServer;
+use Utopia\App;
 use Utopia\Database\Document;
 use Utopia\Database\Helpers\ID;
 use Utopia\Database\Query;
@@ -1569,6 +1570,20 @@ class FunctionsCustomServerTest extends Scope
         $this->assertEquals(200, $response['headers']['status-code']);
         $this->assertEquals($cookie, $response['body']);
 
+        // Await Aggregation
+        sleep(App::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', 30));
+
+        $response = $this->client->call(Client::METHOD_GET, '/functions/' . $functionId . '/usage', array_merge([
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $this->getProject()['$id']
+        ], $this->getHeaders()), [
+            'range' => '24h'
+        ]);
+
+        $this->assertEquals(200, $response['headers']['status-code']);
+        $this->assertEquals(19, count($response['body']));
+        $this->assertEquals('24h', $response['body']['range']);
+
         // Cleanup : Delete function
         $response = $this->client->call(Client::METHOD_DELETE, '/functions/' . $functionId, [
             'content-type' => 'application/json',