Merge branch '1.5.x' of https://github.com/appwrite/appwrite into chore-sync-1.5.x

.env

@@ -1,4 +1,5 @@
 _APP_ENV=development
+_APP_EDITION=self-hosted
 _APP_LOCALE=en
 _APP_WORKER_PER_CORE=6
 _APP_CONSOLE_WHITELIST_ROOT=disabled

.gitignore

@@ -14,3 +14,4 @@ app/sdks
 dev/yasd_init.php
 .phpunit.result.cache
 Makefile
+appwrite.json

.gitmodules

@@ -1,4 +1,4 @@
 [submodule "app/console"]
 	path = app/console
 	url = https://github.com/appwrite/console
-	branch = 4.3.2
+	branch = 4.3.5

CHANGES.md

@@ -1,3 +1,85 @@
+# Version 1.5.7
+## What's Changed
+
+### Fixes
+* Fix database exception wrapping by @abnegate in https://github.com/appwrite/appwrite/pull/7787
+* Fix exception wrap order by @abnegate in https://github.com/appwrite/appwrite/pull/7818
+* Fix membership query to use internalId by @lohanidamodar in https://github.com/appwrite/appwrite/pull/7834
+* Fix vcs silent mode by @vermakhushboo in https://github.com/appwrite/appwrite/pull/7683
+* Fix function domain permissions by @stnguyen90 in https://github.com/appwrite/appwrite/pull/7852
+* Fix tests required for Cloud by @lohanidamodar in https://github.com/appwrite/appwrite/pull/7777
+* Fix OAuth error code by @vermakhushboo in https://github.com/appwrite/appwrite/pull/7893
+* Fix connection reclaim logic. by @eldadfux in https://github.com/appwrite/appwrite/pull/6886
+* Fix shared queue name by @abnegate in https://github.com/appwrite/appwrite/pull/8092
+* Fix syntax error by @abnegate in https://github.com/appwrite/appwrite/pull/8093
+* Fix missing id attribute error by @abnegate in https://github.com/appwrite/appwrite/pull/8094
+* Fix tests for CL by @lohanidamodar in https://github.com/appwrite/appwrite/pull/8076
+* Fix project deletes for shared tables by @abnegate in https://github.com/appwrite/appwrite/pull/8107
+* Handle SQL error code 'HY000' in realtime by @stnguyen90 in https://github.com/appwrite/appwrite/pull/8106
+* Fix: Don't Override `robots.txt` for Other Domains by @ItzNotABug in https://github.com/appwrite/appwrite/pull/8185
+* Escape function build command by @stnguyen90 in https://github.com/appwrite/appwrite/pull/7808
+* Create failed execution from worker if deployment doesn't exist by @vermakhushboo in https://github.com/appwrite/appwrite/pull/7896
+* Fix: admin mode on console by @TorstenDittmann in https://github.com/appwrite/appwrite/pull/7951
+* Fix file size default limit by @shimonewman in https://github.com/appwrite/appwrite/pull/7843
+* Fix: Python failing builds by @Meldiron in https://github.com/appwrite/appwrite/pull/8078
+* Fix shared project delete by @abnegate in https://github.com/appwrite/appwrite/pull/8142
+* Fix TextMagic class name by @stnguyen90 in https://github.com/appwrite/appwrite/pull/8132
+* Prevent functions domain and subdomain to be added as custom domain by @lohanidamodar in https://github.com/appwrite/appwrite/pull/7933
+* Fix don't publish max users exceed by @vermakhushboo in https://github.com/appwrite/appwrite/pull/8067
+* Fix invalid cache document id by @stnguyen90 in https://github.com/appwrite/appwrite/pull/8183
+* Fix not hiding tokens for clients via realtime by @abnegate in https://github.com/appwrite/appwrite/pull/7870
+
+### Miscellaneous
+* Upload 400s to separate error logger by @PineappleIOnic in https://github.com/appwrite/appwrite/pull/7784
+* Admin mode use teamInternalId by @lohanidamodar in https://github.com/appwrite/appwrite/pull/7835
+* Chore: update avatars API by @christyjacob4 in https://github.com/appwrite/appwrite/pull/7840
+* Use internal ids for query by @lohanidamodar in https://github.com/appwrite/appwrite/pull/7838
+* Remove cloud related scripts by @shimonewman in https://github.com/appwrite/appwrite/pull/7414
+* Update VCS Comment by @vermakhushboo in https://github.com/appwrite/appwrite/pull/7854
+* Transaction and reconnection fixes by @fogelito in https://github.com/appwrite/appwrite/pull/7877
+* Feat configurable collections by @christyjacob4 in https://github.com/appwrite/appwrite/pull/7882
+* Remove var_dump calls by @stnguyen90 in https://github.com/appwrite/appwrite/pull/7884
+* Storage DO adapter http version by @lohanidamodar in https://github.com/appwrite/appwrite/pull/7905
+* Update executor version by @vermakhushboo in https://github.com/appwrite/appwrite/pull/7910
+* Comment timer tick by @vermakhushboo in https://github.com/appwrite/appwrite/pull/7911
+* Update db for relationships and object as array attributes fixes by @abnegate in https://github.com/appwrite/appwrite/pull/7917
+* Bump executor version to 0.5.1 by @vermakhushboo in https://github.com/appwrite/appwrite/pull/7925
+* Update database by @abnegate in https://github.com/appwrite/appwrite/pull/7937
+* Reclaim only current connection by @abnegate in https://github.com/appwrite/appwrite/pull/7941
+* Match memberships on internal ID by @abnegate in https://github.com/appwrite/appwrite/pull/7953
+* Chore: queue retry update by @shimonewman in https://github.com/appwrite/appwrite/pull/7991
+* Chore task addition by @shimonewman in https://github.com/appwrite/appwrite/pull/7992
+* Databases.php collection not found by @fogelito in https://github.com/appwrite/appwrite/pull/7341
+* Update database by @abnegate in https://github.com/appwrite/appwrite/pull/8036
+* Feat upgrade db by @abnegate in https://github.com/appwrite/appwrite/pull/8050
+* Handle string error codes by @fogelito in https://github.com/appwrite/appwrite/pull/7878
+* Migration Logging Improvements by @PineappleIOnic in https://github.com/appwrite/appwrite/pull/8057
+* Remove logger code from avatars.php by @vermakhushboo in https://github.com/appwrite/appwrite/pull/8065
+* Update chunk size to 7 MB by @vermakhushboo in https://github.com/appwrite/appwrite/pull/8060
+* Shared tables support by @abnegate in https://github.com/appwrite/appwrite/pull/7206
+* Ensure namespace is set if override equals shared tables by @abnegate in https://github.com/appwrite/appwrite/pull/8091
+* Update database by @abnegate in https://github.com/appwrite/appwrite/pull/8095
+* Disable sending realtime stats by @stnguyen90 in https://github.com/appwrite/appwrite/pull/8104
+* Increase chunk size to 10 MB by @vermakhushboo in https://github.com/appwrite/appwrite/pull/8099
+* Update db by @abnegate in https://github.com/appwrite/appwrite/pull/8113
+* Update executor image name to exc-1 by @vermakhushboo in https://github.com/appwrite/appwrite/pull/8123
+* Catch DB errors on delete by @abnegate in https://github.com/appwrite/appwrite/pull/8143
+* Update Logger and migrations, implement sampler. by @PineappleIOnic in https://github.com/appwrite/appwrite/pull/8146
+* Increase shared tables projects by @abnegate in https://github.com/appwrite/appwrite/pull/8161
+* Feat: improve cold start error, merge to cloud by @loks0n in https://github.com/appwrite/appwrite/pull/8165
+* Add tests for scheduled functions by @vermakhushboo in https://github.com/appwrite/appwrite/pull/8164
+* Remove throw PdoException in Error hook by @fogelito in https://github.com/appwrite/appwrite/pull/8169
+* Refactor localdevice injection by @byawitz in https://github.com/appwrite/appwrite/pull/8173
+* Usage sms per country code count by @shimonewman in https://github.com/appwrite/appwrite/pull/7592
+* GetEnv on worker.php by @shimonewman in https://github.com/appwrite/appwrite/pull/8026
+* Feat get env by @shimonewman in https://github.com/appwrite/appwrite/pull/8180
+* Chore: remove compose version by @loks0n in https://github.com/appwrite/appwrite/pull/8148
+* Chore update executor host default var by @abnegate in https://github.com/appwrite/appwrite/pull/8190
+* Wrap realtime stats in an edition check by @abnegate in https://github.com/appwrite/appwrite/pull/8192
+* Update executor image name by @vermakhushboo in https://github.com/appwrite/appwrite/pull/8147
+* Feat: improve header demo values by @loks0n in https://github.com/appwrite/appwrite/pull/8089
+* Feat: add warning header by @loks0n in https://github.com/appwrite/appwrite/pull/8063
+
 # Version 1.5.6
 ## What's Changed
 

README-CN.md

@@ -67,7 +67,7 @@ docker run -it --rm \
     --volume /var/run/docker.sock:/var/run/docker.sock \
     --volume "$(pwd)"/appwrite:/usr/src/code/appwrite:rw \
     --entrypoint="install" \
-    appwrite/appwrite:1.5.6
+    appwrite/appwrite:1.5.7
 ```
 
 ### Windows
@@ -79,7 +79,7 @@ docker run -it --rm ^
     --volume //var/run/docker.sock:/var/run/docker.sock ^
     --volume "%cd%"/appwrite:/usr/src/code/appwrite:rw ^
     --entrypoint="install" ^
-    appwrite/appwrite:1.5.6
+    appwrite/appwrite:1.5.7
 ```
 
 #### PowerShell
@@ -89,7 +89,7 @@ docker run -it --rm `
     --volume /var/run/docker.sock:/var/run/docker.sock `
     --volume ${pwd}/appwrite:/usr/src/code/appwrite:rw `
     --entrypoint="install" `
-    appwrite/appwrite:1.5.6
+    appwrite/appwrite:1.5.7
 ```
 
 运行后，可以在浏览器上访问 http://localhost 找到 Appwrite 控制台。在非 Linux 的本机主机上完成安装后，服务器可能需要几分钟才能启动。

README.md

@@ -75,7 +75,7 @@ docker run -it --rm \
     --volume /var/run/docker.sock:/var/run/docker.sock \
     --volume "$(pwd)"/appwrite:/usr/src/code/appwrite:rw \
     --entrypoint="install" \
-    appwrite/appwrite:1.5.6
+    appwrite/appwrite:1.5.7
 ```
 
 ### Windows
@@ -87,7 +87,7 @@ docker run -it --rm ^
     --volume //var/run/docker.sock:/var/run/docker.sock ^
     --volume "%cd%"/appwrite:/usr/src/code/appwrite:rw ^
     --entrypoint="install" ^
-    appwrite/appwrite:1.5.6
+    appwrite/appwrite:1.5.7
 ```
 
 #### PowerShell
@@ -97,7 +97,7 @@ docker run -it --rm `
     --volume /var/run/docker.sock:/var/run/docker.sock `
     --volume ${pwd}/appwrite:/usr/src/code/appwrite:rw `
     --entrypoint="install" `
-    appwrite/appwrite:1.5.6
+    appwrite/appwrite:1.5.7
 ```
 
 Once the Docker installation is complete, go to http://localhost to access the Appwrite console from your browser. Please note that on non-Linux native hosts, the server might take a few minutes to start after completing the installation.

app/cli.php

@@ -15,6 +15,7 @@ use Utopia\Config\Config;
 use Utopia\Database\Database;
 use Utopia\Database\Document;
 use Utopia\Database\Validator\Authorization;
+use Utopia\DSN\DSN;
 use Utopia\Logger\Log;
 use Utopia\Platform\Service;
 use Utopia\Pools\Group;
@@ -98,25 +99,53 @@ CLI::setResource('getProjectDB', function (Group $pools, Database $dbForConsole,
             return $dbForConsole;
         }
 
-        $databaseName = $project->getAttribute('database');
+        try {
+            $dsn = new DSN($project->getAttribute('database'));
+        } catch (\InvalidArgumentException) {
+            // TODO: Temporary until all projects are using shared tables
+            $dsn = new DSN('mysql://' . $project->getAttribute('database'));
+        }
+
+        if (isset($databases[$dsn->getHost()])) {
+            $database = $databases[$dsn->getHost()];
+
+            if ($dsn->getHost() === DATABASE_SHARED_TABLES) {
+                $database
+                    ->setSharedTables(true)
+                    ->setTenant($project->getInternalId())
+                    ->setNamespace($dsn->getParam('namespace'));
+            } else {
+                $database
+                    ->setSharedTables(false)
+                    ->setTenant(null)
+                    ->setNamespace('_' . $project->getInternalId());
+            }
 
-        if (isset($databases[$databaseName])) {
-            $database = $databases[$databaseName];
-            $database->setNamespace('_' . $project->getInternalId());
             return $database;
         }
 
         $dbAdapter = $pools
-            ->get($databaseName)
+            ->get($dsn->getHost())
             ->pop()
             ->getResource();
 
         $database = new Database($dbAdapter, $cache);
 
-        $databases[$databaseName] = $database;
+        $databases[$dsn->getHost()] = $database;
+
+        if ($dsn->getHost() === DATABASE_SHARED_TABLES) {
+            $database
+                ->setSharedTables(true)
+                ->setTenant($project->getInternalId())
+                ->setNamespace($dsn->getParam('namespace'));
+        } else {
+            $database
+                ->setSharedTables(false)
+                ->setTenant(null)
+                ->setNamespace('_' . $project->getInternalId());
+        }
 
         $database
-            ->setNamespace('_' . $project->getInternalId())
             ->setMetadata('host', \gethostname())
             ->setMetadata('project', $project->getId());
 

app/config/collections.php

@@ -390,7 +390,7 @@ $commonCollections = [
                 '$id' => ID::custom('_key_email'),
                 'type' => Database::INDEX_UNIQUE,
                 'attributes' => ['email'],
-                'lengths' => [320],
+                'lengths' => [256],
                 'orders' => [Database::ORDER_ASC],
             ],
             [
@@ -996,7 +996,7 @@ $commonCollections = [
                 '$id' => ID::custom('_key_provider_providerUid'),
                 'type' => Database::INDEX_KEY,
                 'attributes' => ['provider', 'providerUid'],
-                'lengths' => [100, 100],
+                'lengths' => [128, 128],
                 'orders' => [Database::ORDER_ASC, Database::ORDER_ASC],
             ],
             [
@@ -1120,14 +1120,14 @@ $commonCollections = [
                 '$id' => ID::custom('_key_userInternalId_provider_providerUid'),
                 'type' => Database::INDEX_UNIQUE,
                 'attributes' => ['userInternalId', 'provider', 'providerUid'],
-                'lengths' => [Database::LENGTH_KEY, 100, 385],
+                'lengths' => [11, 128, 128],
                 'orders' => [Database::ORDER_ASC, Database::ORDER_ASC],
             ],
             [
                 '$id' => ID::custom('_key_provider_providerUid'),
                 'type' => Database::INDEX_UNIQUE,
                 'attributes' => ['provider', 'providerUid'],
-                'lengths' => [100, 640],
+                'lengths' => [128, 128],
                 'orders' => [Database::ORDER_ASC, Database::ORDER_ASC],
             ],
             [
@@ -1148,7 +1148,7 @@ $commonCollections = [
                 '$id' => ID::custom('_key_provider'),
                 'type' => Database::INDEX_KEY,
                 'attributes' => ['provider'],
-                'lengths' => [100],
+                'lengths' => [128],
                 'orders' => [Database::ORDER_ASC],
             ],
             [
@@ -3068,7 +3068,7 @@ $projectCollections = array_merge([
                 '$id' => ID::custom('_key_name'),
                 'type' => Database::INDEX_KEY,
                 'attributes' => ['name'],
-                'lengths' => [768],
+                'lengths' => [256],
                 'orders' => [Database::ORDER_ASC],
             ],
             [
@@ -3117,7 +3117,7 @@ $projectCollections = array_merge([
                 '$id' => ID::custom('_key_runtime'),
                 'type' => Database::INDEX_KEY,
                 'attributes' => ['runtime'],
-                'lengths' => [768],
+                'lengths' => [64],
                 'orders' => [Database::ORDER_ASC],
             ],
             [
@@ -3857,14 +3857,14 @@ $projectCollections = array_merge([
                 '$id' => ID::custom('_key_trigger'),
                 'type' => Database::INDEX_KEY,
                 'attributes' => ['trigger'],
-                'lengths' => [128],
+                'lengths' => [32],
                 'orders' => [Database::ORDER_ASC],
             ],
             [
                 '$id' => ID::custom('_key_status'),
                 'type' => Database::INDEX_KEY,
                 'attributes' => ['status'],
-                'lengths' => [128],
+                'lengths' => [32],
                 'orders' => [Database::ORDER_ASC],
             ],
             [
@@ -5833,14 +5833,14 @@ $bucketCollections = [
                 '$id' => ID::custom('_key_name'),
                 'type' => Database::INDEX_KEY,
                 'attributes' => ['name'],
-                'lengths' => [768],
+                'lengths' => [256],
                 'orders' => [Database::ORDER_ASC],
             ],
             [
                 '$id' => ID::custom('_key_signature'),
                 'type' => Database::INDEX_KEY,
                 'attributes' => ['signature'],
-                'lengths' => [768],
+                'lengths' => [256],
                 'orders' => [Database::ORDER_ASC],
             ],
             [

app/config/errors.php

@@ -134,7 +134,7 @@ return [
     Exception::USER_COUNT_EXCEEDED => [
         'name' => Exception::USER_COUNT_EXCEEDED,
         'description' => 'The current project has exceeded the maximum number of users. Please check your user limit in the Appwrite console.',
-        'code' => 501,
+        'code' => 400,
     ],
     Exception::USER_CONSOLE_COUNT_EXCEEDED => [
         'name' => Exception::USER_CONSOLE_COUNT_EXCEEDED,
@@ -673,6 +673,11 @@ return [
         'description' => 'The attribute type is invalid.',
         'code' => 400,
     ],
+    Exception::RELATIONSHIP_VALUE_INVALID => [
+        'name' => Exception::RELATIONSHIP_VALUE_INVALID,
+        'description' => 'The relationship value is invalid.',
+        'code' => 400,
+    ],
 
     /** Indexes */
     Exception::INDEX_NOT_FOUND => [
@@ -715,7 +720,7 @@ return [
     Exception::PROJECT_PROVIDER_UNSUPPORTED => [
         'name' => Exception::PROJECT_PROVIDER_UNSUPPORTED,
         'description' => 'The chosen OAuth provider is unsupported. Please check the <a href="/docs/client/account?sdk=web-default#accountCreateOAuth2Session">Create OAuth2 Session docs</a> for the complete list of supported OAuth providers.',
-        'code' => 501,
+        'code' => 400,
     ],
     Exception::PROJECT_INVALID_SUCCESS_URL => [
         'name' => Exception::PROJECT_INVALID_SUCCESS_URL,

app/config/variables.php

@@ -756,7 +756,7 @@ return [
             ],
             [
                 'name' => '_APP_EXECUTOR_SECRET',
-                'description' => 'The secret key used by Appwrite to communicate with the function executor. Make sure to change this!',
+                'description' => 'The secret key used by Appwrite to communicate with the function executor. Make sure to change this.',
                 'introduction' => '0.13.0',
                 'default' => 'your-secret-key',
                 'required' => false,
@@ -765,9 +765,9 @@ return [
             ],
             [
                 'name' => '_APP_EXECUTOR_HOST',
-                'description' => 'The host used by Appwrite to communicate with the function executor!',
+                'description' => 'The host used by Appwrite to communicate with the function executor.',
                 'introduction' => '0.13.0',
-                'default' => 'http://appwrite-executor/v1',
+                'default' => 'http://exc1/v1',
                 'required' => false,
                 'overwrite' => true,
                 'question' => '',
@@ -775,7 +775,7 @@ return [
             ],
             [
                 'name' => '_APP_EXECUTOR_RUNTIME_NETWORK',
-                'description' => 'Deprecated with 0.14.0, use \'OPEN_RUNTIMES_NETWORK\' instead!',
+                'description' => 'Deprecated with 0.14.0, use \'OPEN_RUNTIMES_NETWORK\' instead.',
                 'introduction' => '0.13.0',
                 'default' => 'appwrite_runtimes',
                 'required' => false,
@@ -784,7 +784,7 @@ return [
             ],
             [
                 'name' => '_APP_FUNCTIONS_ENVS',
-                'description' => 'Deprecated with 0.8.0, use \'_APP_FUNCTIONS_RUNTIMES\' instead!',
+                'description' => 'Deprecated with 0.8.0, use \'_APP_FUNCTIONS_RUNTIMES\' instead.',
                 'introduction' => '0.7.0',
                 'default' => 'node-16.0,php-7.4,python-3.9,ruby-3.0',
                 'required' => false,
@@ -802,7 +802,7 @@ return [
             ],
             [
                 'name' => 'DOCKERHUB_PULL_USERNAME',
-                'description' => 'Deprecated with 1.2.0, use \'_APP_DOCKER_HUB_USERNAME\' instead!',
+                'description' => 'Deprecated with 1.2.0, use \'_APP_DOCKER_HUB_USERNAME\' instead.',
                 'introduction' => '0.10.0',
                 'default' => '',
                 'required' => false,
@@ -811,7 +811,7 @@ return [
             ],
             [
                 'name' => 'DOCKERHUB_PULL_PASSWORD',
-                'description' => 'Deprecated with 1.2.0, use \'_APP_DOCKER_HUB_PASSWORD\' instead!',
+                'description' => 'Deprecated with 1.2.0, use \'_APP_DOCKER_HUB_PASSWORD\' instead.',
                 'introduction' => '0.10.0',
                 'default' => '',
                 'required' => false,
@@ -829,7 +829,7 @@ return [
             ],
             [
                 'name' => 'OPEN_RUNTIMES_NETWORK',
-                'description' => 'Deprecated with 1.2.0, use \'_APP_FUNCTIONS_RUNTIMES_NETWORK\' instead!',
+                'description' => 'Deprecated with 1.2.0, use \'_APP_FUNCTIONS_RUNTIMES_NETWORK\' instead.',
                 'introduction' => '0.13.0',
                 'default' => 'appwrite_runtimes',
                 'required' => false,

app/console

@@ -1 +1 @@
-Subproject commit d75187458097a20b9e693079b16cb3f25933d31c
+Subproject commit 5169fe16d63066f64ab5013c78953aea04e24b53

app/controllers/api/account.php

@@ -1550,7 +1550,7 @@ App::get('/v1/account/sessions/oauth2/:provider/redirect')
 
                 $target
                     ->setAttribute('sessionId', $session->getId())
-                    ->setAttrubte('sessionInternalId', $session->getInternalId());
+                    ->setAttribute('sessionInternalId', $session->getInternalId());
 
                 $dbForProject->updateDocument('targets', $target->getId(), $target);
             }
@@ -1863,15 +1863,16 @@ App::post('/v1/account/tokens/magic-url')
             ->setRecipient($email)
             ->trigger();
 
-        $queueForEvents->setPayload(
-            $response->output(
-                $token->setAttribute('secret', $tokenSecret),
-                Response::MODEL_TOKEN
-            )
-        );
+        // Set to unhashed secret for events and server responses
+        $token->setAttribute('secret', $tokenSecret);
+
+        $queueForEvents
+            ->setPayload($response->output($token, Response::MODEL_TOKEN), sensitive: ['secret']);
 
         // Hide secret for clients
-        $token->setAttribute('secret', ($isPrivilegedUser || $isAppUser) ? $tokenSecret : '');
+        if (!$isPrivilegedUser && !$isAppUser) {
+            $token->setAttribute('secret', '');
+        }
 
         if (!empty($phrase)) {
             $token->setAttribute('phrase', $phrase);
@@ -1879,8 +1880,7 @@ App::post('/v1/account/tokens/magic-url')
 
         $response
             ->setStatusCode(Response::STATUS_CODE_CREATED)
-            ->dynamic($token, Response::MODEL_TOKEN)
-        ;
+            ->dynamic($token, Response::MODEL_TOKEN);
     });
 
 App::post('/v1/account/tokens/email')
@@ -2092,15 +2092,16 @@ App::post('/v1/account/tokens/email')
             ->setRecipient($email)
             ->trigger();
 
-        $queueForEvents->setPayload(
-            $response->output(
-                $token->setAttribute('secret', $tokenSecret),
-                Response::MODEL_TOKEN
-            )
-        );
+        // Set to unhashed secret for events and server responses
+        $token->setAttribute('secret', $tokenSecret);
+
+        $queueForEvents
+            ->setPayload($response->output($token, Response::MODEL_TOKEN), sensitive: ['secret']);
 
         // Hide secret for clients
-        $token->setAttribute('secret', ($isPrivilegedUser || $isAppUser) ? $tokenSecret : '');
+        if (!$isPrivilegedUser && !$isAppUser) {
+            $token->setAttribute('secret', '');
+        }
 
         if (!empty($phrase)) {
             $token->setAttribute('phrase', $phrase);
@@ -2108,8 +2109,7 @@ App::post('/v1/account/tokens/email')
 
         $response
             ->setStatusCode(Response::STATUS_CODE_CREATED)
-            ->dynamic($token, Response::MODEL_TOKEN)
-        ;
+            ->dynamic($token, Response::MODEL_TOKEN);
     });
 
 App::put('/v1/account/sessions/magic-url')
@@ -2326,20 +2326,18 @@ App::post('/v1/account/tokens/phone')
             ->setRecipients([$phone])
             ->setProviderType(MESSAGE_TYPE_SMS);
 
-        $queueForEvents->setPayload(
-            $response->output(
-                $token->setAttribute('secret', $secret),
-                Response::MODEL_TOKEN
-            )
-        );
+        // Set to unhashed secret for events and server responses
+        $token->setAttribute('secret', $secret);
+
+        $queueForEvents
+            ->setPayload($response->output($token, Response::MODEL_TOKEN), sensitive: ['secret']);
 
         // Hide secret for clients
         $token->setAttribute('secret', ($isPrivilegedUser || $isAppUser) ? Auth::encodeSession($user->getId(), $secret) : '');
 
         $response
             ->setStatusCode(Response::STATUS_CODE_CREATED)
-            ->dynamic($token, Response::MODEL_TOKEN)
-        ;
+            ->dynamic($token, Response::MODEL_TOKEN);
     });
 
 App::post('/v1/account/jwt')
@@ -2623,7 +2621,7 @@ App::patch('/v1/account/email')
         // Makes sure this email is not already used in another identity
         $identityWithMatchingEmail = $dbForProject->findOne('identities', [
             Query::equal('providerEmail', [$email]),
-            Query::notEqual('userId', $user->getId()),
+            Query::notEqual('userInternalId', $user->getInternalId()),
         ]);
         if ($identityWithMatchingEmail !== false && !$identityWithMatchingEmail->isEmpty()) {
             throw new Exception(Exception::GENERAL_BAD_REQUEST); /** Return a generic bad request to prevent exposing existing accounts */
@@ -2987,18 +2985,19 @@ App::post('/v1/account/recovery')
             ->setSubject($subject)
             ->trigger();
 
+        // Set to unhashed secret for events and server responses
+        $recovery->setAttribute('secret', $secret);
+
         $queueForEvents
             ->setParam('userId', $profile->getId())
             ->setParam('tokenId', $recovery->getId())
             ->setUser($profile)
-            ->setPayload($response->output(
-                $recovery->setAttribute('secret', $secret),
-                Response::MODEL_TOKEN
-            ))
-        ;
+            ->setPayload($response->output($recovery, Response::MODEL_TOKEN), sensitive: ['secret']);
 
         // Hide secret for clients
-        $recovery->setAttribute('secret', ($isPrivilegedUser || $isAppUser) ? $secret : '');
+        if (!$isPrivilegedUser && !$isAppUser) {
+            $recovery->setAttribute('secret', '');
+        }
 
         $response
             ->setStatusCode(Response::STATUS_CODE_CREATED)
@@ -3169,6 +3168,7 @@ App::post('/v1/account/verification')
             ->setParam('{{footer}}', $locale->getText("emails.verification.footer"))
             ->setParam('{{thanks}}', $locale->getText("emails.verification.thanks"))
             ->setParam('{{signature}}', $locale->getText("emails.verification.signature"));
+
         $body = $message->render();
 
         $smtp = $project->getAttribute('smtp', []);
@@ -3235,16 +3235,18 @@ App::post('/v1/account/verification')
             ->setName($user->getAttribute('name') ?? '')
             ->trigger();
 
+        // Set to unhashed secret for events and server responses
+        $verification->setAttribute('secret', $verificationSecret);
+
         $queueForEvents
             ->setParam('userId', $user->getId())
             ->setParam('tokenId', $verification->getId())
-            ->setPayload($response->output(
-                $verification->setAttribute('secret', $verificationSecret),
-                Response::MODEL_TOKEN
-            ));
+            ->setPayload($response->output($verification, Response::MODEL_TOKEN), sensitive: ['secret']);
 
         // Hide secret for clients
-        $verification->setAttribute('secret', ($isPrivilegedUser || $isAppUser) ? $verificationSecret : '');
+        if (!$isPrivilegedUser && !$isAppUser) {
+            $verification->setAttribute('secret', '');
+        }
 
         $response
             ->setStatusCode(Response::STATUS_CODE_CREATED)
@@ -3294,7 +3296,7 @@ App::put('/v1/account/verification')
 
         $user->setAttributes($profile->getArrayCopy());
 
-        $verificationDocument = $dbForProject->getDocument('tokens', $verifiedToken->getId());
+        $verification = $dbForProject->getDocument('tokens', $verifiedToken->getId());
 
         /**
          * We act like we're updating and validating
@@ -3305,10 +3307,9 @@ App::put('/v1/account/verification')
 
         $queueForEvents
             ->setParam('userId', $userId)
-            ->setParam('tokenId', $verificationDocument->getId())
-        ;
+            ->setParam('tokenId', $verification->getId());
 
-        $response->dynamic($verificationDocument, Response::MODEL_TOKEN);
+        $response->dynamic($verification, Response::MODEL_TOKEN);
     });
 
 App::post('/v1/account/verification/phone')
@@ -3406,17 +3407,18 @@ App::post('/v1/account/verification/phone')
             ->setRecipients([$user->getAttribute('phone')])
             ->setProviderType(MESSAGE_TYPE_SMS);
 
+        // Set to unhashed secret for events and server responses
+        $verification->setAttribute('secret', $secret);
+
         $queueForEvents
             ->setParam('userId', $user->getId())
             ->setParam('tokenId', $verification->getId())
-            ->setPayload($response->output(
-                $verification->setAttribute('secret', $secret),
-                Response::MODEL_TOKEN
-            ))
-        ;
+            ->setPayload($response->output($verification, Response::MODEL_TOKEN), sensitive: ['secret']);
 
         // Hide secret for clients
-        $verification->setAttribute('secret', ($isPrivilegedUser || $isAppUser) ? $secret : '');
+        if (!$isPrivilegedUser && !$isAppUser) {
+            $verification->setAttribute('secret', '');
+        }
 
         $response
             ->setStatusCode(Response::STATUS_CODE_CREATED)

app/controllers/api/avatars.php

@@ -6,7 +6,6 @@ use Appwrite\Utopia\Response;
 use chillerlan\QRCode\QRCode;
 use chillerlan\QRCode\QROptions;
 use Utopia\App;
-use Utopia\CLI\Console;
 use Utopia\Config\Config;
 use Utopia\Database\Database;
 use Utopia\Database\DateTime;
@@ -14,8 +13,8 @@ use Utopia\Database\Document;
 use Utopia\Database\Validator\Authorization;
 use Utopia\Database\Validator\UID;
 use Utopia\Domains\Domain;
+use Utopia\Fetch\Client;
 use Utopia\Image\Image;
-use Utopia\Logger\Log;
 use Utopia\Logger\Logger;
 use Utopia\System\System;
 use Utopia\Validator\Boolean;
@@ -155,40 +154,8 @@ $getUserGitHub = function (string $userId, Document $project, Database $dbForPro
             'id' => $githubId
         ];
     } catch (Exception $error) {
-        if ($logger) {
-            $version = System::getEnv('_APP_VERSION', 'UNKNOWN');
-
-            $log = new Log();
-            $log->setNamespace('console');
-            $log->setServer(\gethostname());
-            $log->setVersion($version);
-            $log->setType(Log::TYPE_ERROR);
-            $log->setMessage($error->getMessage());
-
-            $log->addTag('code', $error->getCode());
-            $log->addTag('verboseType', get_class($error));
-
-            $log->addExtra('file', $error->getFile());
-            $log->addExtra('line', $error->getLine());
-            $log->addExtra('trace', $error->getTraceAsString());
-            $log->addExtra('detailedTrace', $error->getTrace());
-
-            $log->setAction('avatarsGetGitHub');
-
-            $isProduction = System::getEnv('_APP_ENV', 'development') === 'production';
-            $log->setEnvironment($isProduction ? Log::ENVIRONMENT_PRODUCTION : Log::ENVIRONMENT_STAGING);
-
-            $responseCode = $logger->addLog($log);
-            Console::info('GitHub error log pushed with status code: ' . $responseCode);
-        }
-
-        Console::warning("Failed: {$error->getMessage()}");
-        Console::warning($error->getTraceAsString());
-
         return [];
     }
-
-    return [];
 };
 
 App::get('/v1/avatars/credit-cards/:code')
@@ -284,14 +251,21 @@ App::get('/v1/avatars/image')
             throw new Exception(Exception::AVATAR_REMOTE_URL_FAILED);
         }
 
-        $fetch = @\file_get_contents($url);
+        $client = new Client();
+        try {
+            $res = $client
+                ->setAllowRedirects(false)
+                ->fetch($url);
+        } catch (\Throwable) {
+            throw new Exception(Exception::AVATAR_REMOTE_URL_FAILED);
+        }
 
-        if (!$fetch) {
+        if ($res->getStatusCode() !== 200) {
             throw new Exception(Exception::AVATAR_IMAGE_NOT_FOUND);
         }
 
         try {
-            $image = new Image($fetch);
+            $image = new Image($res->getBody());
         } catch (\Throwable $exception) {
             throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Unable to parse image');
         }
@@ -340,31 +314,27 @@ App::get('/v1/avatars/favicon')
             throw new Exception(Exception::AVATAR_REMOTE_URL_FAILED);
         }
 
-        $curl = \curl_init();
+        $client = new Client();
+        try {
+            $res = $client
+                ->setAllowRedirects(false)
+                ->setUserAgent(\sprintf(
+                    APP_USERAGENT,
+                    System::getEnv('_APP_VERSION', 'UNKNOWN'),
+                    System::getEnv('_APP_SYSTEM_SECURITY_EMAIL_ADDRESS', APP_EMAIL_SECURITY)
+                ))
+                ->fetch($url);
+        } catch (\Throwable) {
+            throw new Exception(Exception::AVATAR_REMOTE_URL_FAILED);
+        }
 
-        \curl_setopt_array($curl, [
-            CURLOPT_RETURNTRANSFER => 1,
-            CURLOPT_FOLLOWLOCATION => true,
-            CURLOPT_MAXREDIRS => 3,
-            CURLOPT_URL => $url,
-            CURLOPT_USERAGENT => \sprintf(
-                APP_USERAGENT,
-                System::getEnv('_APP_VERSION', 'UNKNOWN'),
-                System::getEnv('_APP_SYSTEM_SECURITY_EMAIL_ADDRESS', APP_EMAIL_SECURITY)
-            ),
-        ]);
-
-        $html = \curl_exec($curl);
-
-        \curl_close($curl);
-
-        if (!$html) {
+        if ($res->getStatusCode() !== 200) {
             throw new Exception(Exception::AVATAR_REMOTE_URL_FAILED);
         }
 
         $doc = new DOMDocument();
         $doc->strictErrorChecking = false;
-        @$doc->loadHTML($html);
+        @$doc->loadHTML($res->getBody());
 
         $links = $doc->getElementsByTagName('link');
         $outputHref = '';
@@ -419,9 +389,22 @@ App::get('/v1/avatars/favicon')
             throw new Exception(Exception::AVATAR_REMOTE_URL_FAILED);
         }
 
-        if ('ico' == $outputExt) { // Skip crop, Imagick isn\'t supporting icon files
-            $data = @\file_get_contents($outputHref, false);
+        $client = new Client();
+        try {
+            $res = $client
+                ->setAllowRedirects(false)
+                ->fetch($outputHref);
+        } catch (\Throwable) {
+            throw new Exception(Exception::AVATAR_REMOTE_URL_FAILED);
+        }
 
+        if ($res->getStatusCode() !== 200) {
+            throw new Exception(Exception::AVATAR_ICON_NOT_FOUND);
+        }
+
+        $data = $res->getBody();
+
+        if ('ico' == $outputExt) { // Skip crop, Imagick isn\'t supporting icon files
             if (empty($data) || (\mb_substr($data, 0, 5) === '<html') || \mb_substr($data, 0, 5) === '<!doc') {
                 throw new Exception(Exception::AVATAR_ICON_NOT_FOUND, 'Favicon not found');
             }
@@ -431,13 +414,7 @@ App::get('/v1/avatars/favicon')
                 ->file($data);
         }
 
-        $fetch = @\file_get_contents($outputHref, false);
-
-        if (!$fetch) {
-            throw new Exception(Exception::AVATAR_ICON_NOT_FOUND);
-        }
-
-        $image = new Image($fetch);
+        $image = new Image($data);
         $image->crop((int) $width, (int) $height);
         $output = (empty($output)) ? $type : $output;
         $data = $image->output($output, $quality);

app/controllers/api/databases.php

@@ -24,7 +24,6 @@ use Utopia\Database\Exception\Authorization as AuthorizationException;
 use Utopia\Database\Exception\Conflict as ConflictException;
 use Utopia\Database\Exception\Duplicate as DuplicateException;
 use Utopia\Database\Exception\Limit as LimitException;
-use Utopia\Database\Exception\Query as QueryException;
 use Utopia\Database\Exception\Restricted as RestrictedException;
 use Utopia\Database\Exception\Structure as StructureException;
 use Utopia\Database\Helpers\ID;
@@ -666,16 +665,10 @@ App::put('/v1/databases/:databaseId')
             throw new Exception(Exception::DATABASE_NOT_FOUND);
         }
 
-        try {
-            $database = $dbForProject->updateDocument('databases', $databaseId, $database
-                ->setAttribute('name', $name)
-                ->setAttribute('enabled', $enabled)
-                ->setAttribute('search', implode(' ', [$databaseId, $name])));
-        } catch (AuthorizationException) {
-            throw new Exception(Exception::USER_UNAUTHORIZED);
-        } catch (StructureException $exception) {
-            throw new Exception(Exception::DOCUMENT_INVALID_STRUCTURE, 'Bad structure. ' . $exception->getMessage());
-        }
+        $database = $dbForProject->updateDocument('databases', $databaseId, $database
+            ->setAttribute('name', $name)
+            ->setAttribute('enabled', $enabled)
+            ->setAttribute('search', implode(' ', [$databaseId, $name])));
 
         $queueForEvents->setParam('databaseId', $database->getId());
 
@@ -1036,19 +1029,14 @@ App::put('/v1/databases/:databaseId/collections/:collectionId')
 
         $enabled ??= $collection->getAttribute('enabled', true);
 
-        try {
-            $collection = $dbForProject->updateDocument('database_' . $database->getInternalId(), $collectionId, $collection
-                ->setAttribute('name', $name)
-                ->setAttribute('$permissions', $permissions)
-                ->setAttribute('documentSecurity', $documentSecurity)
-                ->setAttribute('enabled', $enabled)
-                ->setAttribute('search', implode(' ', [$collectionId, $name])));
-            $dbForProject->updateCollection('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $permissions, $documentSecurity);
-        } catch (AuthorizationException) {
-            throw new Exception(Exception::USER_UNAUTHORIZED);
-        } catch (StructureException $exception) {
-            throw new Exception(Exception::DOCUMENT_INVALID_STRUCTURE, 'Bad structure. ' . $exception->getMessage());
-        }
+        $collection = $dbForProject->updateDocument('database_' . $database->getInternalId(), $collectionId, $collection
+            ->setAttribute('name', $name)
+            ->setAttribute('$permissions', $permissions)
+            ->setAttribute('documentSecurity', $documentSecurity)
+            ->setAttribute('enabled', $enabled)
+            ->setAttribute('search', implode(' ', [$collectionId, $name])));
+
+        $dbForProject->updateCollection('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $permissions, $documentSecurity);
 
         $queueForEvents
             ->setContext('database', $database)
@@ -3599,16 +3587,10 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu
         }
 
         $dbForProject->withRequestTimestamp($requestTimestamp, function () use ($dbForProject, $database, $collection, $documentId) {
-            try {
-                $dbForProject->deleteDocument(
-                    'database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(),
-                    $documentId
-                );
-            } catch (AuthorizationException) {
-                throw new Exception(Exception::USER_UNAUTHORIZED);
-            } catch (RestrictedException) {
-                throw new Exception(Exception::DOCUMENT_DELETE_RESTRICTED);
-            }
+            $dbForProject->deleteDocument(
+                'database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(),
+                $documentId
+            );
         });
 
         // Add $collectionId and $databaseId for all documents

app/controllers/api/projects.php

@@ -5,11 +5,13 @@ use Appwrite\Event\Delete;
 use Appwrite\Event\Mail;
 use Appwrite\Event\Validator\Event;
 use Appwrite\Extend\Exception;
+use Appwrite\Hooks\Hooks;
 use Appwrite\Network\Validator\Email;
 use Appwrite\Network\Validator\Origin;
 use Appwrite\Template\Template;
 use Appwrite\Utopia\Database\Validator\ProjectId;
 use Appwrite\Utopia\Database\Validator\Queries\Projects;
+use Appwrite\Utopia\Request;
 use Appwrite\Utopia\Response;
 use PHPMailer\PHPMailer\PHPMailer;
 use Utopia\Abuse\Adapters\TimeLimit;
@@ -28,6 +30,7 @@ use Utopia\Database\Query;
 use Utopia\Database\Validator\Datetime as DatetimeValidator;
 use Utopia\Database\Validator\UID;
 use Utopia\Domains\Validator\PublicDomain;
+use Utopia\DSN\DSN;
 use Utopia\Locale\Locale;
 use Utopia\Pools\Group;
 use Utopia\System\System;
@@ -73,11 +76,13 @@ App::post('/v1/projects')
     ->param('legalCity', '', new Text(256), 'Project legal City. Max length: 256 chars.', true)
     ->param('legalAddress', '', new Text(256), 'Project legal Address. Max length: 256 chars.', true)
     ->param('legalTaxId', '', new Text(256), 'Project legal Tax ID. Max length: 256 chars.', true)
+    ->inject('request')
     ->inject('response')
     ->inject('dbForConsole')
     ->inject('cache')
     ->inject('pools')
-    ->action(function (string $projectId, string $name, string $teamId, string $region, string $description, string $logo, string $url, string $legalName, string $legalCountry, string $legalState, string $legalCity, string $legalAddress, string $legalTaxId, Response $response, Database $dbForConsole, Cache $cache, Group $pools) {
+    ->inject('hooks')
+    ->action(function (string $projectId, string $name, string $teamId, string $region, string $description, string $logo, string $url, string $legalName, string $legalCountry, string $legalState, string $legalCity, string $legalAddress, string $legalTaxId, Request $request, Response $response, Database $dbForConsole, Cache $cache, Group $pools, Hooks $hooks) {
 
         $team = $dbForConsole->getDocument('teams', $teamId);
 
@@ -92,8 +97,15 @@ App::post('/v1/projects')
         }
 
         $auth = Config::getParam('auth', []);
-        $auths = ['limit' => 0, 'maxSessions' => APP_LIMIT_USER_SESSIONS_DEFAULT, 'passwordHistory' => 0, 'passwordDictionary' => false, 'duration' => Auth::TOKEN_EXPIRATION_LOGIN_LONG, 'personalDataCheck' => false];
-        foreach ($auth as $index => $method) {
+        $auths = [
+            'limit' => 0,
+            'maxSessions' => APP_LIMIT_USER_SESSIONS_DEFAULT,
+            'passwordHistory' => 0,
+            'passwordDictionary' => false,
+            'duration' => Auth::TOKEN_EXPIRATION_LOGIN_LONG,
+            'personalDataCheck' => false
+        ];
+        foreach ($auth as $method) {
             $auths[$method['key'] ?? ''] = true;
         }
 
@@ -128,18 +140,55 @@ App::post('/v1/projects')
             }
         }
 
-        $databaseOverride = System::getEnv('_APP_DATABASE_OVERRIDE', null);
-        $index = array_search($databaseOverride, $databases);
+        $databaseOverride = System::getEnv('_APP_DATABASE_OVERRIDE');
+        $index = \array_search($databaseOverride, $databases);
         if ($index !== false) {
-            $database = $databases[$index];
+            $dsn = $databases[$index];
         } else {
-            $database = $databases[array_rand($databases)];
+            $dsn = $databases[array_rand($databases)];
         }
 
         if ($projectId === 'console') {
             throw new Exception(Exception::PROJECT_RESERVED_PROJECT, "'console' is a reserved project.");
         }
 
+        // TODO: 1 in 5 projects use shared tables. Temporary until all projects are using shared tables.
+        if (
+            (
+                !\mt_rand(0, 4)
+                && System::getEnv('_APP_DATABASE_SHARED_TABLES', 'enabled') === 'enabled'
+                && System::getEnv('_APP_EDITION', 'self-hosted') !== 'self-hosted'
+            ) ||
+            (
+                $dsn === DATABASE_SHARED_TABLES
+            )
+        ) {
+            $schema = 'appwrite';
+            $database = 'appwrite';
+            $namespace = System::getEnv('_APP_DATABASE_SHARED_NAMESPACE', '');
+            $dsn = $schema . '://' . DATABASE_SHARED_TABLES . '?database=' . $database;
+
+            if (!empty($namespace)) {
+                $dsn .= '&namespace=' . $namespace;
+            }
+        }
+
+        // TODO: Allow overriding in development mode. Temporary until all projects are using shared tables.
+        if (
+            App::isDevelopment()
+            && System::getEnv('_APP_EDITION', 'self-hosted') !== 'self-hosted'
+            && $request->getHeader('x-appwrited-share-tables', false)
+        ) {
+            $schema = 'appwrite';
+            $database = 'appwrite';
+            $namespace = System::getEnv('_APP_DATABASE_SHARED_NAMESPACE', '');
+            $dsn = $schema . '://' . DATABASE_SHARED_TABLES . '?database=' . $database;
+
+            if (!empty($namespace)) {
+                $dsn .= '&namespace=' . $namespace;
+            }
+        }
+
         try {
             $project = $dbForConsole->createDocument('projects', new Document([
                 '$id' => $projectId,
@@ -171,21 +220,41 @@ App::post('/v1/projects')
                 'keys' => null,
                 'auths' => $auths,
                 'search' => implode(' ', [$projectId, $name]),
-                'database' => $database
+                'database' => $dsn,
             ]));
-        } catch (Duplicate $th) {
+        } catch (Duplicate) {
             throw new Exception(Exception::PROJECT_ALREADY_EXISTS);
         }
 
-        $dbForProject = new Database($pools->get($database)->pop()->getResource(), $cache);
-        $dbForProject->setNamespace("_{$project->getInternalId()}");
+        try {
+            $dsn = new DSN($dsn);
+        } catch (\InvalidArgumentException) {
+            // TODO: Temporary until all projects are using shared tables
+            $dsn = new DSN('mysql://' . $dsn);
+        }
+
+        $adapter = $pools->get($dsn->getHost())->pop()->getResource();
+        $dbForProject = new Database($adapter, $cache);
+
+        if ($dsn->getHost() === DATABASE_SHARED_TABLES) {
+            $dbForProject
+                ->setSharedTables(true)
+                ->setTenant($project->getInternalId())
+                ->setNamespace($dsn->getParam('namespace'));
+        } else {
+            $dbForProject
+                ->setSharedTables(false)
+                ->setTenant(null)
+                ->setNamespace('_' . $project->getInternalId());
+        }
+
         $dbForProject->create();
 
         $audit = new Audit($dbForProject);
         $audit->setup();
 
-        $adapter = new TimeLimit('', 0, 1, $dbForProject);
-        $adapter->setup();
+        $abuse = new TimeLimit('', 0, 1, $dbForProject);
+        $abuse->setup();
 
         /** @var array $collections */
         $collections = Config::getParam('collections', [])['projects'] ?? [];
@@ -195,35 +264,25 @@ App::post('/v1/projects')
                 continue;
             }
 
-            $attributes = [];
-            $indexes = [];
+            $attributes = \array_map(function (array $attribute) {
+                return new Document($attribute);
+            }, $collection['attributes']);
 
-            foreach ($collection['attributes'] as $attribute) {
-                $attributes[] = new Document([
-                    '$id' => $attribute['$id'],
-                    'type' => $attribute['type'],
-                    'size' => $attribute['size'],
-                    'required' => $attribute['required'],
-                    'signed' => $attribute['signed'],
-                    'array' => $attribute['array'],
-                    'filters' => $attribute['filters'],
-                    'default' => $attribute['default'] ?? null,
-                    'format' => $attribute['format'] ?? ''
-                ]);
-            }
+            $indexes = \array_map(function (array $index) {
+                return new Document($index);
+            }, $collection['indexes']);
 
-            foreach ($collection['indexes'] as $index) {
-                $indexes[] = new Document([
-                    '$id' => $index['$id'],
-                    'type' => $index['type'],
-                    'attributes' => $index['attributes'],
-                    'lengths' => $index['lengths'],
-                    'orders' => $index['orders'],
-                ]);
+            try {
+                $dbForProject->createCollection($key, $attributes, $indexes);
+            } catch (Duplicate) {
+                // Collection already exists
             }
-            $dbForProject->createCollection($key, $attributes, $indexes);
         }
 
+        // Hook allowing instant project mirroring during migration
+        // Outside of migration, hook is not registered and has no effect
+        $hooks->trigger('afterProjectCreation', [ $project, $pools, $cache ]);
+
         $response
             ->setStatusCode(Response::STATUS_CODE_CREATED)
             ->dynamic($project, Response::MODEL_PROJECT);

app/controllers/api/storage.php

@@ -16,11 +16,8 @@ use Utopia\App;
 use Utopia\Config\Config;
 use Utopia\Database\Database;
 use Utopia\Database\Document;
-use Utopia\Database\Exception\Authorization as AuthorizationException;
 use Utopia\Database\Exception\Duplicate;
-use Utopia\Database\Exception\Duplicate as DuplicateException;
 use Utopia\Database\Exception\Query as QueryException;
-use Utopia\Database\Exception\Structure as StructureException;
 use Utopia\Database\Helpers\ID;
 use Utopia\Database\Helpers\Permission;
 use Utopia\Database\Helpers\Role;
@@ -66,7 +63,7 @@ App::post('/v1/storage/buckets')
     ->param('permissions', null, new Permissions(APP_LIMIT_ARRAY_PARAMS_SIZE), 'An array of permission strings. By default, no user is granted with any permissions. [Learn more about permissions](https://appwrite.io/docs/permissions).', true)
     ->param('fileSecurity', false, new Boolean(true), 'Enables configuring permissions for individual file. A user needs one of file or bucket level permissions to access a file. [Learn more about permissions](https://appwrite.io/docs/permissions).', true)
     ->param('enabled', true, new Boolean(true), 'Is bucket enabled? When set to \'disabled\', users cannot access the files in this bucket but Server SDKs with and API key can still access the bucket. No files are lost when this is toggled.', true)
-    ->param('maximumFileSize', (int) System::getEnv('_APP_STORAGE_LIMIT', 0), new Range(1, (int) System::getEnv('_APP_STORAGE_LIMIT', 0)), 'Maximum file size allowed in bytes. Maximum allowed value is ' . Storage::human(System::getEnv('_APP_STORAGE_LIMIT', 0), 0) . '.', true)
+    ->param('maximumFileSize', fn (array $plan) => empty($plan['fileSize']) ? (int) System::getEnv('_APP_STORAGE_LIMIT', 0) : $plan['fileSize'] * 1024 * 1024, fn (array $plan) => new Range(1, empty($plan['fileSize']) ? (int) System::getEnv('_APP_STORAGE_LIMIT', 0) : $plan['fileSize'] * 1024 * 1024 * 1024), 'Maximum file size allowed in bytes. Maximum allowed value is ' . Storage::human(System::getEnv('_APP_STORAGE_LIMIT', 0), 0) . '.', true, ['plan'])
     ->param('allowedFileExtensions', [], new ArrayList(new Text(64), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Allowed file extensions. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' extensions are allowed, each 64 characters long.', true)
     ->param('compression', Compression::NONE, new WhiteList([Compression::NONE, Compression::GZIP, Compression::ZSTD]), 'Compression algorithm choosen for compression. Can be one of ' . Compression::NONE . ',  [' . Compression::GZIP . '](https://en.wikipedia.org/wiki/Gzip), or [' . Compression::ZSTD . '](https://en.wikipedia.org/wiki/Zstd), For file size above ' . Storage::human(APP_STORAGE_READ_BUFFER, 0) . ' compression is skipped even if it\'s enabled', true)
     ->param('encryption', true, new Boolean(true), 'Is encryption enabled? For file size above ' . Storage::human(APP_STORAGE_READ_BUFFER, 0) . ' encryption is skipped even if it\'s enabled', true)
@@ -588,111 +585,95 @@ App::post('/v1/storage/buckets/:bucketId/files')
                 $openSSLIV = \bin2hex($iv);
             }
 
-            try {
-                if ($file->isEmpty()) {
-                    $doc = new Document([
-                        '$id' => $fileId,
-                        '$permissions' => $permissions,
-                        'bucketId' => $bucket->getId(),
-                        'bucketInternalId' => $bucket->getInternalId(),
-                        'name' => $fileName,
-                        'path' => $path,
-                        'signature' => $fileHash,
-                        'mimeType' => $mimeType,
-                        'sizeOriginal' => $fileSize,
-                        'sizeActual' => $sizeActual,
-                        'algorithm' => $algorithm,
-                        'comment' => '',
-                        'chunksTotal' => $chunks,
-                        'chunksUploaded' => $chunksUploaded,
-                        'openSSLVersion' => $openSSLVersion,
-                        'openSSLCipher' => $openSSLCipher,
-                        'openSSLTag' => $openSSLTag,
-                        'openSSLIV' => $openSSLIV,
-                        'search' => implode(' ', [$fileId, $fileName]),
-                        'metadata' => $metadata,
-                    ]);
+            if ($file->isEmpty()) {
+                $doc = new Document([
+                    '$id' => $fileId,
+                    '$permissions' => $permissions,
+                    'bucketId' => $bucket->getId(),
+                    'bucketInternalId' => $bucket->getInternalId(),
+                    'name' => $fileName,
+                    'path' => $path,
+                    'signature' => $fileHash,
+                    'mimeType' => $mimeType,
+                    'sizeOriginal' => $fileSize,
+                    'sizeActual' => $sizeActual,
+                    'algorithm' => $algorithm,
+                    'comment' => '',
+                    'chunksTotal' => $chunks,
+                    'chunksUploaded' => $chunksUploaded,
+                    'openSSLVersion' => $openSSLVersion,
+                    'openSSLCipher' => $openSSLCipher,
+                    'openSSLTag' => $openSSLTag,
+                    'openSSLIV' => $openSSLIV,
+                    'search' => implode(' ', [$fileId, $fileName]),
+                    'metadata' => $metadata,
+                ]);
 
-                    $file = $dbForProject->createDocument('bucket_' . $bucket->getInternalId(), $doc);
-                } else {
-                    $file = $file
-                        ->setAttribute('$permissions', $permissions)
-                        ->setAttribute('signature', $fileHash)
-                        ->setAttribute('mimeType', $mimeType)
-                        ->setAttribute('sizeActual', $sizeActual)
-                        ->setAttribute('algorithm', $algorithm)
-                        ->setAttribute('openSSLVersion', $openSSLVersion)
-                        ->setAttribute('openSSLCipher', $openSSLCipher)
-                        ->setAttribute('openSSLTag', $openSSLTag)
-                        ->setAttribute('openSSLIV', $openSSLIV)
-                        ->setAttribute('metadata', $metadata)
-                        ->setAttribute('chunksUploaded', $chunksUploaded);
+                $file = $dbForProject->createDocument('bucket_' . $bucket->getInternalId(), $doc);
+            } else {
+                $file = $file
+                    ->setAttribute('$permissions', $permissions)
+                    ->setAttribute('signature', $fileHash)
+                    ->setAttribute('mimeType', $mimeType)
+                    ->setAttribute('sizeActual', $sizeActual)
+                    ->setAttribute('algorithm', $algorithm)
+                    ->setAttribute('openSSLVersion', $openSSLVersion)
+                    ->setAttribute('openSSLCipher', $openSSLCipher)
+                    ->setAttribute('openSSLTag', $openSSLTag)
+                    ->setAttribute('openSSLIV', $openSSLIV)
+                    ->setAttribute('metadata', $metadata)
+                    ->setAttribute('chunksUploaded', $chunksUploaded);
 
-                    /**
-                     * Validate create permission and skip authorization in updateDocument
-                     * Without this, the file creation will fail when user doesn't have update permission
-                     * However as with chunk upload even if we are updating, we are essentially creating a file
-                     * adding it's new chunk so we validate create permission instead of update
-                     */
-                    $validator = new Authorization(Database::PERMISSION_CREATE);
-                    if (!$validator->isValid($bucket->getCreate())) {
-                        throw new Exception(Exception::USER_UNAUTHORIZED);
-                    }
-                    $file = Authorization::skip(fn () => $dbForProject->updateDocument('bucket_' . $bucket->getInternalId(), $fileId, $file));
+                /**
+                 * Validate create permission and skip authorization in updateDocument
+                 * Without this, the file creation will fail when user doesn't have update permission
+                 * However as with chunk upload even if we are updating, we are essentially creating a file
+                 * adding it's new chunk so we validate create permission instead of update
+                 */
+                $validator = new Authorization(Database::PERMISSION_CREATE);
+                if (!$validator->isValid($bucket->getCreate())) {
+                    throw new Exception(Exception::USER_UNAUTHORIZED);
                 }
-            } catch (AuthorizationException) {
-                throw new Exception(Exception::USER_UNAUTHORIZED);
-            } catch (StructureException $exception) {
-                throw new Exception(Exception::DOCUMENT_INVALID_STRUCTURE, $exception->getMessage());
-            } catch (DuplicateException) {
-                throw new Exception(Exception::DOCUMENT_ALREADY_EXISTS);
+                $file = Authorization::skip(fn () => $dbForProject->updateDocument('bucket_' . $bucket->getInternalId(), $fileId, $file));
             }
         } else {
-            try {
-                if ($file->isEmpty()) {
-                    $doc = new Document([
-                        '$id' => ID::custom($fileId),
-                        '$permissions' => $permissions,
-                        'bucketId' => $bucket->getId(),
-                        'bucketInternalId' => $bucket->getInternalId(),
-                        'name' => $fileName,
-                        'path' => $path,
-                        'signature' => '',
-                        'mimeType' => '',
-                        'sizeOriginal' => $fileSize,
-                        'sizeActual' => 0,
-                        'algorithm' => '',
-                        'comment' => '',
-                        'chunksTotal' => $chunks,
-                        'chunksUploaded' => $chunksUploaded,
-                        'search' => implode(' ', [$fileId, $fileName]),
-                        'metadata' => $metadata,
-                    ]);
+            if ($file->isEmpty()) {
+                $doc = new Document([
+                    '$id' => ID::custom($fileId),
+                    '$permissions' => $permissions,
+                    'bucketId' => $bucket->getId(),
+                    'bucketInternalId' => $bucket->getInternalId(),
+                    'name' => $fileName,
+                    'path' => $path,
+                    'signature' => '',
+                    'mimeType' => '',
+                    'sizeOriginal' => $fileSize,
+                    'sizeActual' => 0,
+                    'algorithm' => '',
+                    'comment' => '',
+                    'chunksTotal' => $chunks,
+                    'chunksUploaded' => $chunksUploaded,
+                    'search' => implode(' ', [$fileId, $fileName]),
+                    'metadata' => $metadata,
+                ]);
 
-                    $file = $dbForProject->createDocument('bucket_' . $bucket->getInternalId(), $doc);
-                } else {
-                    $file = $file
-                        ->setAttribute('chunksUploaded', $chunksUploaded)
-                        ->setAttribute('metadata', $metadata);
+                $file = $dbForProject->createDocument('bucket_' . $bucket->getInternalId(), $doc);
+            } else {
+                $file = $file
+                    ->setAttribute('chunksUploaded', $chunksUploaded)
+                    ->setAttribute('metadata', $metadata);
 
-                    /**
-                     * Validate create permission and skip authorization in updateDocument
-                     * Without this, the file creation will fail when user doesn't have update permission
-                     * However as with chunk upload even if we are updating, we are essentially creating a file
-                     * adding it's new chunk so we validate create permission instead of update
-                     */
-                    $validator = new Authorization(Database::PERMISSION_CREATE);
-                    if (!$validator->isValid($bucket->getCreate())) {
-                        throw new Exception(Exception::USER_UNAUTHORIZED);
-                    }
-                    $file = Authorization::skip(fn () => $dbForProject->updateDocument('bucket_' . $bucket->getInternalId(), $fileId, $file));
+                /**
+                 * Validate create permission and skip authorization in updateDocument
+                 * Without this, the file creation will fail when user doesn't have update permission
+                 * However as with chunk upload even if we are updating, we are essentially creating a file
+                 * adding it's new chunk so we validate create permission instead of update
+                 */
+                $validator = new Authorization(Database::PERMISSION_CREATE);
+                if (!$validator->isValid($bucket->getCreate())) {
+                    throw new Exception(Exception::USER_UNAUTHORIZED);
                 }
-            } catch (AuthorizationException) {
-                throw new Exception(Exception::USER_UNAUTHORIZED);
-            } catch (StructureException $exception) {
-                throw new Exception(Exception::DOCUMENT_INVALID_STRUCTURE, $exception->getMessage());
-            } catch (DuplicateException) {
-                throw new Exception(Exception::DOCUMENT_ALREADY_EXISTS);
+                $file = Authorization::skip(fn () => $dbForProject->updateDocument('bucket_' . $bucket->getInternalId(), $fileId, $file));
             }
         }
 
@@ -1554,11 +1535,7 @@ App::put('/v1/storage/buckets/:bucketId/files/:fileId')
         }
 
         if ($fileSecurity && !$valid) {
-            try {
-                $file = $dbForProject->updateDocument('bucket_' . $bucket->getInternalId(), $fileId, $file);
-            } catch (AuthorizationException) {
-                throw new Exception(Exception::USER_UNAUTHORIZED);
-            }
+            $file = $dbForProject->updateDocument('bucket_' . $bucket->getInternalId(), $fileId, $file);
         } else {
             $file = Authorization::skip(fn () => $dbForProject->updateDocument('bucket_' . $bucket->getInternalId(), $fileId, $file));
         }
@@ -1643,11 +1620,7 @@ App::delete('/v1/storage/buckets/:bucketId/files/:fileId')
             ;
 
             if ($fileSecurity && !$valid) {
-                try {
-                    $deleted = $dbForProject->deleteDocument('bucket_' . $bucket->getInternalId(), $fileId);
-                } catch (AuthorizationException) {
-                    throw new Exception(Exception::USER_UNAUTHORIZED);
-                }
+                $deleted = $dbForProject->deleteDocument('bucket_' . $bucket->getInternalId(), $fileId);
             } else {
                 $deleted = Authorization::skip(fn () => $dbForProject->deleteDocument('bucket_' . $bucket->getInternalId(), $fileId));
             }

app/controllers/api/vcs.php

@@ -102,7 +102,7 @@ $createGitDeployments = function (GitHub $github, string $providerInstallationId
 
             $latestCommentId = '';
 
-            if (!empty($providerPullRequestId)) {
+            if (!empty($providerPullRequestId) && $function->getAttribute('providerSilentMode', false) === false) {
                 $latestComment = Authorization::skip(fn () => $dbForConsole->findOne('vcsComments', [
                     Query::equal('providerRepositoryId', [$providerRepositoryId]),
                     Query::equal('providerPullRequestId', [$providerPullRequestId]),

app/controllers/general.php

@@ -26,6 +26,7 @@ use Utopia\Database\Helpers\ID;
 use Utopia\Database\Query;
 use Utopia\Database\Validator\Authorization;
 use Utopia\Domains\Domain;
+use Utopia\DSN\DSN;
 use Utopia\Locale\Locale;
 use Utopia\Logger\Log;
 use Utopia\Logger\Log\User;
@@ -362,6 +363,31 @@ function router(App $utopia, Database $dbForConsole, callable $getProjectDB, Swo
     return false;
 }
 
+/*
+App::init()
+    ->groups(['api'])
+    ->inject('project')
+    ->inject('mode')
+    ->action(function (Document $project, string $mode) {
+        if ($mode === APP_MODE_ADMIN && $project->getId() === 'console') {
+            throw new AppwriteException(AppwriteException::GENERAL_BAD_REQUEST, 'Admin mode is not allowed for console project');
+        }
+    });
+*/
+
+App::init()
+    ->groups(['database', 'functions', 'storage', 'messaging'])
+    ->inject('project')
+    ->inject('request')
+    ->action(function (Document $project, Request $request) {
+        if ($project->getId() === 'console') {
+            $message = empty($request->getHeader('x-appwrite-project')) ?
+                'No Appwrite project was specified. Please specify your project ID when initializing your Appwrite SDK.' :
+                'This endpoint is not available for the console project. The Appwrite Console is a reserved project ID and cannot be used with the Appwrite SDKs and APIs. Please check if your project ID is correct.';
+            throw new AppwriteException(AppwriteException::GENERAL_ACCESS_FORBIDDEN, $message);
+        }
+    });
+
 App::init()
     ->groups(['api', 'web'])
     ->inject('utopia')
@@ -399,7 +425,9 @@ App::init()
         Request::setRoute($route);
 
         if ($route === null) {
-            return $response->setStatusCode(404)->send('Not Found');
+            return $response
+                ->setStatusCode(404)
+                ->send('Not Found');
         }
 
         $requestFormat = $request->getHeader('x-appwrite-response-format', System::getEnv('_APP_SYSTEM_RESPONSE_FORMAT', ''));
@@ -526,6 +554,9 @@ App::init()
             if (version_compare($responseFormat, '1.5.0', '<')) {
                 $response->addFilter(new ResponseV17());
             }
+            if (version_compare($responseFormat, APP_VERSION_STABLE, '>')) {
+                $response->addHeader('X-Appwrite-Warning', "The current SDK is built for Appwrite " . $responseFormat . ". However, the current Appwrite server version is ". APP_VERSION_STABLE . ". Please downgrade your SDK to match the Appwrite version: https://appwrite.io/docs/sdks");
+            }
         }
 
         /*
@@ -552,7 +583,7 @@ App::init()
             ->addHeader('Server', 'Appwrite')
             ->addHeader('X-Content-Type-Options', 'nosniff')
             ->addHeader('Access-Control-Allow-Methods', 'GET, POST, PUT, PATCH, DELETE')
-            ->addHeader('Access-Control-Allow-Headers', 'Origin, Cookie, Set-Cookie, X-Requested-With, Content-Type, Access-Control-Allow-Origin, Access-Control-Request-Headers, Accept, X-Appwrite-Project, X-Appwrite-Key, X-Appwrite-Locale, X-Appwrite-Mode, X-Appwrite-JWT, X-Appwrite-Response-Format, X-Appwrite-Timeout, X-SDK-Version, X-SDK-Name, X-SDK-Language, X-SDK-Platform, X-SDK-GraphQL, X-Appwrite-ID, X-Appwrite-Timestamp, Content-Range, Range, Cache-Control, Expires, Pragma, X-Forwarded-For, X-Forwarded-User-Agent')
+            ->addHeader('Access-Control-Allow-Headers', 'Origin, Cookie, Set-Cookie, X-Requested-With, Content-Type, Access-Control-Allow-Origin, Access-Control-Request-Headers, Accept, X-Appwrite-Project, X-Appwrite-Key, X-Appwrite-Locale, X-Appwrite-Mode, X-Appwrite-JWT, X-Appwrite-Response-Format, X-Appwrite-Timeout, X-Appwrite-Shared-Tables, X-SDK-Version, X-SDK-Name, X-SDK-Language, X-SDK-Platform, X-SDK-GraphQL, X-Appwrite-ID, X-Appwrite-Timestamp, Content-Range, Range, Cache-Control, Expires, Pragma, X-Forwarded-For, X-Forwarded-User-Agent')
             ->addHeader('Access-Control-Expose-Headers', 'X-Appwrite-Session, X-Fallback-Cookies')
             ->addHeader('Access-Control-Allow-Origin', $refDomain)
             ->addHeader('Access-Control-Allow-Credentials', 'true');
@@ -603,7 +634,7 @@ App::options()
         $response
             ->addHeader('Server', 'Appwrite')
             ->addHeader('Access-Control-Allow-Methods', 'GET, POST, PUT, PATCH, DELETE')
-            ->addHeader('Access-Control-Allow-Headers', 'Origin, Cookie, Set-Cookie, X-Requested-With, Content-Type, Access-Control-Allow-Origin, Access-Control-Request-Headers, Accept, X-Appwrite-Project, X-Appwrite-Key, X-Appwrite-Locale, X-Appwrite-Mode, X-Appwrite-JWT, X-Appwrite-Response-Format, X-Appwrite-Timeout, X-SDK-Version, X-SDK-Name, X-SDK-Language, X-SDK-Platform, X-SDK-GraphQL, X-Appwrite-ID, X-Appwrite-Timestamp, Content-Range, Range, Cache-Control, Expires, Pragma, X-Appwrite-Session, X-Fallback-Cookies, X-Forwarded-For, X-Forwarded-User-Agent')
+            ->addHeader('Access-Control-Allow-Headers', 'Origin, Cookie, Set-Cookie, X-Requested-With, Content-Type, Access-Control-Allow-Origin, Access-Control-Request-Headers, Accept, X-Appwrite-Project, X-Appwrite-Key, X-Appwrite-Locale, X-Appwrite-Mode, X-Appwrite-JWT, X-Appwrite-Response-Format, X-Appwrite-Timeout, X-Appwrite-Shared-Tables, X-SDK-Version, X-SDK-Name, X-SDK-Language, X-SDK-Platform, X-SDK-GraphQL, X-Appwrite-ID, X-Appwrite-Timestamp, Content-Range, Range, Cache-Control, Expires, Pragma, X-Appwrite-Session, X-Fallback-Cookies, X-Forwarded-For, X-Forwarded-User-Agent')
             ->addHeader('Access-Control-Expose-Headers', 'X-Appwrite-Session, X-Fallback-Cookies')
             ->addHeader('Access-Control-Allow-Origin', $origin)
             ->addHeader('Access-Control-Allow-Credentials', 'true')
@@ -621,6 +652,67 @@ App::error()
     ->action(function (Throwable $error, App $utopia, Request $request, Response $response, Document $project, ?Logger $logger, Log $log) {
         $version = System::getEnv('_APP_VERSION', 'UNKNOWN');
         $route = $utopia->getRoute();
+        $class = \get_class($error);
+        $code = $error->getCode();
+        $message = $error->getMessage();
+        $file = $error->getFile();
+        $line = $error->getLine();
+        $trace = $error->getTrace();
+
+        if (php_sapi_name() === 'cli') {
+            Console::error('[Error] Timestamp: ' . date('c', time()));
+
+            if ($route) {
+                Console::error('[Error] Method: ' . $route->getMethod());
+                Console::error('[Error] URL: ' . $route->getPath());
+            }
+
+            Console::error('[Error] Type: ' . get_class($error));
+            Console::error('[Error] Message: ' . $message);
+            Console::error('[Error] File: ' . $file);
+            Console::error('[Error] Line: ' . $line);
+        }
+
+        switch ($class) {
+            case 'Utopia\Exception':
+                $error = new AppwriteException(AppwriteException::GENERAL_UNKNOWN, $message, $code, $error);
+                switch ($code) {
+                    case 400:
+                        $error->setType(AppwriteException::GENERAL_ARGUMENT_INVALID);
+                        break;
+                    case 404:
+                        $error->setType(AppwriteException::GENERAL_ROUTE_NOT_FOUND);
+                        break;
+                }
+                break;
+            case 'Utopia\Database\Exception\Conflict':
+                $error = new AppwriteException(AppwriteException::DOCUMENT_UPDATE_CONFLICT, previous: $error);
+                break;
+            case 'Utopia\Database\Exception\Timeout':
+                $error = new AppwriteException(AppwriteException::DATABASE_TIMEOUT, previous: $error);
+                break;
+            case 'Utopia\Database\Exception\Query':
+                $error = new AppwriteException(AppwriteException::GENERAL_QUERY_INVALID, $error->getMessage(), previous: $error);
+                break;
+            case 'Utopia\Database\Exception\Structure':
+                $error = new AppwriteException(AppwriteException::DOCUMENT_INVALID_STRUCTURE, $error->getMessage(), previous: $error);
+                break;
+            case 'Utopia\Database\Exception\Duplicate':
+                $error = new AppwriteException(AppwriteException::DOCUMENT_ALREADY_EXISTS);
+                break;
+            case 'Utopia\Database\Exception\Restricted':
+                $error = new AppwriteException(AppwriteException::DOCUMENT_DELETE_RESTRICTED);
+                break;
+            case 'Utopia\Database\Exception\Authorization':
+                $error = new AppwriteException(AppwriteException::USER_UNAUTHORIZED);
+                break;
+            case 'Utopia\Database\Exception\Relationship':
+                $error = new AppwriteException(AppwriteException::RELATIONSHIP_VALUE_INVALID, $error->getMessage(), previous: $error);
+                break;
+        }
+
+        $code = $error->getCode();
+        $message = $error->getMessage();
 
         if ($error instanceof AppwriteException) {
             $publish = $error->isPublishable();
@@ -628,11 +720,29 @@ App::error()
             $publish = $error->getCode() === 0 || $error->getCode() >= 500;
         }
 
-        if ($logger && ($publish || $error->getCode() === 0)) {
+        if ($error->getCode() >= 400 && $error->getCode() < 500) {
+            // Register error logger
+            $providerName = System::getEnv('_APP_EXPERIMENT_LOGGING_PROVIDER', '');
+            $providerConfig = System::getEnv('_APP_EXPERIMENT_LOGGING_CONFIG', '');
+
+            if (!(empty($providerName) || empty($providerConfig))) {
+                if (!Logger::hasProvider($providerName)) {
+                    throw new Exception("Logging provider not supported. Logging is disabled");
+                }
+
+                $classname = '\\Utopia\\Logger\\Adapter\\' . \ucfirst($providerName);
+                $adapter = new $classname($providerConfig);
+                $logger = new Logger($adapter);
+                $logger->setSample(0.04);
+                $publish = true;
+            }
+        }
+
+        if ($logger && $publish) {
             try {
                 /** @var Utopia\Database\Document $user */
                 $user = $utopia->getResource('user');
-            } catch (\Throwable $th) {
+            } catch (\Throwable) {
                 // All good, user is optional information for logger
             }
 
@@ -640,13 +750,20 @@ App::error()
                 $log->setUser(new User($user->getId()));
             }
 
+            try {
+                $dsn = new DSN($project->getAttribute('database', 'console'));
+            } catch (\InvalidArgumentException) {
+                // TODO: Temporary until all projects are using shared tables
+                $dsn = new DSN('mysql://' . $project->getAttribute('database', 'console'));
+            }
+
             $log->setNamespace("http");
             $log->setServer(\gethostname());
             $log->setVersion($version);
             $log->setType(Log::TYPE_ERROR);
             $log->setMessage($error->getMessage());
 
-            $log->addTag('database', $project->getAttribute('database', 'console'));
+            $log->addTag('database', $dsn->getHost());
             $log->addTag('method', $route->getMethod());
             $log->addTag('url', $route->getPath());
             $log->addTag('verboseType', get_class($error));
@@ -671,47 +788,6 @@ App::error()
             Console::info('Log pushed with status code: ' . $responseCode);
         }
 
-        $code = $error->getCode();
-        $message = $error->getMessage();
-        $file = $error->getFile();
-        $line = $error->getLine();
-        $trace = $error->getTrace();
-
-        if (php_sapi_name() === 'cli') {
-            Console::error('[Error] Timestamp: ' . date('c', time()));
-
-            if ($route) {
-                Console::error('[Error] Method: ' . $route->getMethod());
-                Console::error('[Error] URL: ' . $route->getPath());
-            }
-
-            Console::error('[Error] Type: ' . get_class($error));
-            Console::error('[Error] Message: ' . $message);
-            Console::error('[Error] File: ' . $file);
-            Console::error('[Error] Line: ' . $line);
-        }
-
-        /** Handle Utopia Errors */
-        if ($error instanceof Utopia\Exception) {
-            $error = new AppwriteException(AppwriteException::GENERAL_UNKNOWN, $message, $code, $error);
-            switch ($code) {
-                case 400:
-                    $error->setType(AppwriteException::GENERAL_ARGUMENT_INVALID);
-                    break;
-                case 404:
-                    $error->setType(AppwriteException::GENERAL_ROUTE_NOT_FOUND);
-                    break;
-            }
-        } elseif ($error instanceof Utopia\Database\Exception\Conflict) {
-            $error = new AppwriteException(AppwriteException::DOCUMENT_UPDATE_CONFLICT, previous: $error);
-            $code = $error->getCode();
-            $message = $error->getMessage();
-        } elseif ($error instanceof Utopia\Database\Exception\Timeout) {
-            $error = new AppwriteException(AppwriteException::DATABASE_TIMEOUT, previous: $error);
-            $code = $error->getCode();
-            $message = $error->getMessage();
-        }
-
         /** Wrap all exceptions inside Appwrite\Extend\Exception */
         if (!($error instanceof AppwriteException)) {
             $error = new AppwriteException(AppwriteException::GENERAL_UNKNOWN, $message, $code, $error);
@@ -790,20 +866,50 @@ App::get('/robots.txt')
     ->desc('Robots.txt File')
     ->label('scope', 'public')
     ->label('docs', false)
+    ->inject('utopia')
+    ->inject('swooleRequest')
+    ->inject('request')
     ->inject('response')
-    ->action(function (Response $response) {
-        $template = new View(__DIR__ . '/../views/general/robots.phtml');
-        $response->text($template->render(false));
+    ->inject('dbForConsole')
+    ->inject('getProjectDB')
+    ->inject('queueForEvents')
+    ->inject('queueForUsage')
+    ->inject('geodb')
+    ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Database $dbForConsole, callable $getProjectDB, Event $queueForEvents, Usage $queueForUsage, Reader $geodb) {
+        $host = $request->getHostname() ?? '';
+        $mainDomain = System::getEnv('_APP_DOMAIN', '');
+
+        if ($host === $mainDomain) {
+            $template = new View(__DIR__ . '/../views/general/robots.phtml');
+            $response->text($template->render(false));
+        } else {
+            router($utopia, $dbForConsole, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForUsage, $geodb);
+        }
     });
 
 App::get('/humans.txt')
     ->desc('Humans.txt File')
     ->label('scope', 'public')
     ->label('docs', false)
+    ->inject('utopia')
+    ->inject('swooleRequest')
+    ->inject('request')
     ->inject('response')
-    ->action(function (Response $response) {
-        $template = new View(__DIR__ . '/../views/general/humans.phtml');
-        $response->text($template->render(false));
+    ->inject('dbForConsole')
+    ->inject('getProjectDB')
+    ->inject('queueForEvents')
+    ->inject('queueForUsage')
+    ->inject('geodb')
+    ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Database $dbForConsole, callable $getProjectDB, Event $queueForEvents, Usage $queueForUsage, Reader $geodb) {
+        $host = $request->getHostname() ?? '';
+        $mainDomain = System::getEnv('_APP_DOMAIN', '');
+
+        if ($host === $mainDomain) {
+            $template = new View(__DIR__ . '/../views/general/humans.phtml');
+            $response->text($template->render(false));
+        } else {
+            router($utopia, $dbForConsole, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForUsage, $geodb);
+        }
     });
 
 App::get('/.well-known/acme-challenge/*')

app/controllers/shared/api.php

@@ -586,7 +586,7 @@ App::shutdown()
 
                 Realtime::send(
                     projectId: $target['projectId'] ?? $project->getId(),
-                    payload: $queueForEvents->getPayload(),
+                    payload: $queueForEvents->getRealtimePayload(),
                     events: $allEvents,
                     channels: $target['channels'],
                     roles: $target['roles'],

app/http.php

@@ -32,7 +32,7 @@ $http = new Server(
     mode: SWOOLE_PROCESS,
 );
 
-$payloadSize = 6 * (1024 * 1024); // 6MB
+$payloadSize = 12 * (1024 * 1024); // 12MB - adding slight buffer for headers and other data that might be sent with the payload - update later with valid testing
 $workerNumber = swoole_cpu_num() * intval(System::getEnv('_APP_WORKER_PER_CORE', 6));
 
 $http

app/init.php

@@ -12,11 +12,11 @@ if (\file_exists(__DIR__ . '/../vendor/autoload.php')) {
     require_once __DIR__ . '/../vendor/autoload.php';
 }
 
-ini_set('memory_limit', '512M');
-ini_set('display_errors', 1);
-ini_set('display_startup_errors', 1);
-ini_set('default_socket_timeout', -1);
-error_reporting(E_ALL);
+\ini_set('memory_limit', '512M');
+\ini_set('display_errors', 1);
+\ini_set('display_startup_errors', 1);
+\ini_set('default_socket_timeout', -1);
+\error_reporting(E_ALL);
 
 use Ahc\Jwt\JWT;
 use Ahc\Jwt\JWTException;
@@ -112,8 +112,8 @@ const APP_LIMIT_LIST_DEFAULT = 25; // Default maximum number of items to return
 const APP_KEY_ACCCESS = 24 * 60 * 60; // 24 hours
 const APP_USER_ACCCESS = 24 * 60 * 60; // 24 hours
 const APP_CACHE_UPDATE = 24 * 60 * 60; // 24 hours
-const APP_CACHE_BUSTER = 432;
-const APP_VERSION_STABLE = '1.5.6';
+const APP_CACHE_BUSTER = 443;
+const APP_VERSION_STABLE = '1.5.7';
 const APP_DATABASE_ATTRIBUTE_EMAIL = 'email';
 const APP_DATABASE_ATTRIBUTE_ENUM = 'enum';
 const APP_DATABASE_ATTRIBUTE_IP = 'ip';
@@ -143,6 +143,9 @@ const APP_SOCIAL_STACKSHARE = 'https://stackshare.io/appwrite';
 const APP_SOCIAL_YOUTUBE = 'https://www.youtube.com/c/appwrite?sub_confirmation=1';
 const APP_HOSTNAME_INTERNAL = 'appwrite';
 
+// Databases
+const DATABASE_SHARED_TABLES = 'database_db_fra1_self_hosted_16_0';
+
 // Database Reconnect
 const DATABASE_RECONNECT_SLEEP = 2;
 const DATABASE_RECONNECT_MAX_ATTEMPTS = 10;
@@ -200,7 +203,7 @@ const APP_AUTH_TYPE_JWT = 'JWT';
 const APP_AUTH_TYPE_KEY = 'Key';
 const APP_AUTH_TYPE_ADMIN = 'Admin';
 // Response related
-const MAX_OUTPUT_CHUNK_SIZE = 2 * 1024 * 1024; // 2MB
+const MAX_OUTPUT_CHUNK_SIZE = 10 * 1024 * 1024; // 10MB
 // Function headers
 const FUNCTION_ALLOWLIST_HEADERS_REQUEST = ['content-type', 'agent', 'content-length', 'host'];
 const FUNCTION_ALLOWLIST_HEADERS_RESPONSE = ['content-type', 'content-length'];
@@ -212,6 +215,7 @@ const MESSAGE_TYPE_PUSH = 'push';
 const METRIC_TEAMS = 'teams';
 const METRIC_USERS = 'users';
 const METRIC_MESSAGES  = 'messages';
+const METRIC_MESSAGES_COUNTRY_CODE  = '{countryCode}.messages';
 const METRIC_SESSIONS  = 'sessions';
 const METRIC_DATABASES = 'databases';
 const METRIC_COLLECTIONS = 'collections';
@@ -732,6 +736,16 @@ $register->set('logger', function () {
         throw new Exception(Exception::GENERAL_SERVER_ERROR, "Logging provider not supported. Logging is disabled");
     }
 
+    // Old Sentry Format conversion. Fallback until the old syntax is completely deprecated.
+    if (str_contains($providerConfig, ';') && strtolower($providerName) == 'sentry') {
+        $configChunks = \explode(";", $providerConfig);
+
+        $sentryKey = $configChunks[0];
+        $projectId = $configChunks[1];
+
+        $providerConfig = 'https://' . $sentryKey . '@sentry.io/' . $projectId;
+    }
+
     $classname = '\\Utopia\\Logger\\Adapter\\' . \ucfirst($providerName);
     $adapter = new $classname($providerConfig);
     return new Logger($adapter);
@@ -807,14 +821,13 @@ $register->set('pools', function () {
 
     foreach ($connections as $key => $connection) {
         $type = $connection['type'] ?? '';
-        $dsns = $connection['dsns'] ?? '';
-        $multipe = $connection['multiple'] ?? false;
+        $multiple = $connection['multiple'] ?? false;
         $schemes = $connection['schemes'] ?? [];
         $config = [];
         $dsns = explode(',', $connection['dsns'] ?? '');
         foreach ($dsns as &$dsn) {
             $dsn = explode('=', $dsn);
-            $name = ($multipe) ? $key . '_' . $dsn[0] : $key;
+            $name = ($multiple) ? $key . '_' . $dsn[0] : $key;
             $dsn = $dsn[1] ?? '';
             $config[] = $name;
             if (empty($dsn)) {
@@ -841,38 +854,31 @@ $register->set('pools', function () {
              *
              * Resource assignment to an adapter will happen below.
              */
-            switch ($dsnScheme) {
-                case 'mysql':
-                case 'mariadb':
-                    $resource = function () use ($dsnHost, $dsnPort, $dsnUser, $dsnPass, $dsnDatabase) {
-                        return new PDOProxy(function () use ($dsnHost, $dsnPort, $dsnUser, $dsnPass, $dsnDatabase) {
-                            return new PDO("mysql:host={$dsnHost};port={$dsnPort};dbname={$dsnDatabase};charset=utf8mb4", $dsnUser, $dsnPass, array(
-                                // No need to set PDO::ATTR_ERRMODE it is overwritten in PDOProxy
-                                PDO::ATTR_TIMEOUT => 3, // Seconds
-                                PDO::ATTR_PERSISTENT => true,
-                                PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC,
-                                PDO::ATTR_EMULATE_PREPARES => true,
-                                PDO::ATTR_STRINGIFY_FETCHES => true
-                            ));
-                        });
-                    };
-                    break;
-                case 'redis':
-                    $resource = function () use ($dsnHost, $dsnPort, $dsnPass) {
-                        $redis = new Redis();
-                        @$redis->pconnect($dsnHost, (int)$dsnPort);
-                        if ($dsnPass) {
-                            $redis->auth($dsnPass);
-                        }
-                        $redis->setOption(Redis::OPT_READ_TIMEOUT, -1);
+            $resource = match ($dsnScheme) {
+                'mysql',
+                'mariadb' => function () use ($dsnHost, $dsnPort, $dsnUser, $dsnPass, $dsnDatabase) {
+                    return new PDOProxy(function () use ($dsnHost, $dsnPort, $dsnUser, $dsnPass, $dsnDatabase) {
+                        return new PDO("mysql:host={$dsnHost};port={$dsnPort};dbname={$dsnDatabase};charset=utf8mb4", $dsnUser, $dsnPass, array(
+                            PDO::ATTR_TIMEOUT => 3, // Seconds
+                            PDO::ATTR_PERSISTENT => true,
+                            PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC,
+                            PDO::ATTR_EMULATE_PREPARES => true,
+                            PDO::ATTR_STRINGIFY_FETCHES => true
+                        ));
+                    });
+                },
+                'redis' => function () use ($dsnHost, $dsnPort, $dsnPass) {
+                    $redis = new Redis();
+                    @$redis->pconnect($dsnHost, (int)$dsnPort);
+                    if ($dsnPass) {
+                        $redis->auth($dsnPass);
+                    }
+                    $redis->setOption(Redis::OPT_READ_TIMEOUT, -1);
 
-                        return $redis;
-                    };
-                    break;
-
-                default:
-                    throw new Exception(Exception::GENERAL_SERVER_ERROR, "Invalid scheme");
-            }
+                    return $redis;
+                },
+                default => throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Invalid scheme'),
+            };
 
             $pool = new Pool($name, $poolSize, function () use ($type, $resource, $dsn) {
                 // Get Adapter
@@ -1096,24 +1102,25 @@ App::setResource('clients', function ($request, $console, $project) {
         fn ($node) => $node['hostname'],
         \array_filter(
             $console->getAttribute('platforms', []),
-            fn ($node) => (isset($node['type']) && ($node['type'] === Origin::CLIENT_TYPE_WEB) && isset($node['hostname']) && !empty($node['hostname']))
+            fn ($node) => (isset($node['type']) && ($node['type'] === Origin::CLIENT_TYPE_WEB) && !empty($node['hostname']))
         )
     );
 
-    $clients = \array_unique(
-        \array_merge(
-            $clientsConsole,
-            \array_map(
-                fn ($node) => $node['hostname'],
-                \array_filter(
-                    $project->getAttribute('platforms', []),
-                    fn ($node) => (isset($node['type']) && ($node['type'] === Origin::CLIENT_TYPE_WEB || $node['type'] === Origin::CLIENT_TYPE_FLUTTER_WEB) && isset($node['hostname']) && !empty($node['hostname']))
-                )
-            )
-        )
-    );
+    $clients = $clientsConsole;
+    $platforms = $project->getAttribute('platforms', []);
 
-    return $clients;
+    foreach ($platforms as $node) {
+        if (
+            isset($node['type']) &&
+            ($node['type'] === Origin::CLIENT_TYPE_WEB ||
+            $node['type'] === Origin::CLIENT_TYPE_FLUTTER_WEB) &&
+            !empty($node['hostname'])
+        ) {
+            $clients[] = $node['hostname'];
+        }
+    }
+
+    return \array_unique($clients);
 }, ['request', 'console', 'project']);
 
 App::setResource('user', function ($mode, $project, $console, $request, $response, $dbForProject, $dbForConsole) {
@@ -1187,7 +1194,7 @@ App::setResource('user', function ($mode, $project, $console, $request, $respons
     }
 
     if (APP_MODE_ADMIN === $mode) {
-        if ($user->find('teamId', $project->getAttribute('teamId'), 'memberships')) {
+        if ($user->find('teamInternalId', $project->getAttribute('teamInternalId'), 'memberships')) {
             Authorization::setDefaultStatus(false);  // Cancel security segmentation for admin users.
         } else {
             $user = new Document([]);
@@ -1305,19 +1312,44 @@ App::setResource('dbForProject', function (Group $pools, Database $dbForConsole,
         return $dbForConsole;
     }
 
+    try {
+        $dsn = new DSN($project->getAttribute('database'));
+    } catch (\InvalidArgumentException) {
+        // TODO: Temporary until all projects are using shared tables
+        $dsn = new DSN('mysql://' . $project->getAttribute('database'));
+    }
+
     $dbAdapter = $pools
-        ->get($project->getAttribute('database'))
+        ->get($dsn->getHost())
         ->pop()
         ->getResource();
 
     $database = new Database($dbAdapter, $cache);
 
     $database
-        ->setNamespace('_' . $project->getInternalId())
         ->setMetadata('host', \gethostname())
         ->setMetadata('project', $project->getId())
         ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS);
 
+    try {
+        $dsn = new DSN($project->getAttribute('database'));
+    } catch (\InvalidArgumentException) {
+        // TODO: Temporary until all projects are using shared tables
+        $dsn = new DSN('mysql://' . $project->getAttribute('database'));
+    }
+
+    if ($dsn->getHost() === DATABASE_SHARED_TABLES) {
+        $database
+            ->setSharedTables(true)
+            ->setTenant($project->getInternalId())
+            ->setNamespace($dsn->getParam('namespace'));
+    } else {
+        $database
+            ->setSharedTables(false)
+            ->setTenant(null)
+            ->setNamespace('_' . $project->getInternalId());
+    }
+
     return $database;
 }, ['pools', 'dbForConsole', 'cache', 'project']);
 
@@ -1325,8 +1357,7 @@ App::setResource('dbForConsole', function (Group $pools, Cache $cache) {
     $dbAdapter = $pools
         ->get('console')
         ->pop()
-        ->getResource()
-    ;
+        ->getResource();
 
     $database = new Database($dbAdapter, $cache);
 
@@ -1342,44 +1373,54 @@ App::setResource('dbForConsole', function (Group $pools, Cache $cache) {
 App::setResource('getProjectDB', function (Group $pools, Database $dbForConsole, $cache) {
     $databases = []; // TODO: @Meldiron This should probably be responsibility of utopia-php/pools
 
-    $getProjectDB = function (Document $project) use ($pools, $dbForConsole, $cache, &$databases) {
+    return function (Document $project) use ($pools, $dbForConsole, $cache, &$databases) {
         if ($project->isEmpty() || $project->getId() === 'console') {
             return $dbForConsole;
         }
 
-        $databaseName = $project->getAttribute('database');
-
-        if (isset($databases[$databaseName])) {
-            $database = $databases[$databaseName];
+        try {
+            $dsn = new DSN($project->getAttribute('database'));
+        } catch (\InvalidArgumentException) {
+            // TODO: Temporary until all projects are using shared tables
+            $dsn = new DSN('mysql://' . $project->getAttribute('database'));
+        }
 
+        $configure = (function (Database $database) use ($project, $dsn) {
             $database
-                ->setNamespace('_' . $project->getInternalId())
                 ->setMetadata('host', \gethostname())
                 ->setMetadata('project', $project->getId())
                 ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS);
 
+            if ($dsn->getHost() === DATABASE_SHARED_TABLES) {
+                $database
+                    ->setSharedTables(true)
+                    ->setTenant($project->getInternalId())
+                    ->setNamespace($dsn->getParam('namespace'));
+            } else {
+                $database
+                    ->setSharedTables(false)
+                    ->setTenant(null)
+                    ->setNamespace('_' . $project->getInternalId());
+            }
+        });
+
+        if (isset($databases[$dsn->getHost()])) {
+            $database = $databases[$dsn->getHost()];
+            $configure($database);
             return $database;
         }
 
         $dbAdapter = $pools
-            ->get($databaseName)
+            ->get($dsn->getHost())
             ->pop()
             ->getResource();
 
         $database = new Database($dbAdapter, $cache);
-
-        $databases[$databaseName] = $database;
-
-        $database
-            ->setNamespace('_' . $project->getInternalId())
-            ->setMetadata('host', \gethostname())
-            ->setMetadata('project', $project->getId())
-            ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS);
+        $databases[$dsn->getHost()] = $database;
+        $configure($database);
 
         return $database;
     };
-
-    return $getProjectDB;
 }, ['pools', 'dbForConsole', 'cache']);
 
 App::setResource('cache', function (Group $pools) {
@@ -1440,7 +1481,9 @@ function getDevice($root): Device
             case Storage::DEVICE_S3:
                 return new S3($root, $accessKey, $accessSecret, $bucket, $region, $acl);
             case STORAGE::DEVICE_DO_SPACES:
-                return new DOSpaces($root, $accessKey, $accessSecret, $bucket, $region, $acl);
+                $device = new DOSpaces($root, $accessKey, $accessSecret, $bucket, $region, $acl);
+                $device->setHttpVersion(S3::HTTP_VERSION_1_1);
+                return $device;
             case Storage::DEVICE_BACKBLAZE:
                 return new Backblaze($root, $accessKey, $accessSecret, $bucket, $region, $acl);
             case Storage::DEVICE_LINODE:
@@ -1469,7 +1512,9 @@ function getDevice($root): Device
                 $doSpacesRegion = System::getEnv('_APP_STORAGE_DO_SPACES_REGION', '');
                 $doSpacesBucket = System::getEnv('_APP_STORAGE_DO_SPACES_BUCKET', '');
                 $doSpacesAcl = 'private';
-                return new DOSpaces($root, $doSpacesAccessKey, $doSpacesSecretKey, $doSpacesBucket, $doSpacesRegion, $doSpacesAcl);
+                $device = new DOSpaces($root, $doSpacesAccessKey, $doSpacesSecretKey, $doSpacesBucket, $doSpacesRegion, $doSpacesAcl);
+                $device->setHttpVersion(S3::HTTP_VERSION_1_1);
+                return $device;
             case Storage::DEVICE_BACKBLAZE:
                 $backblazeAccessKey = System::getEnv('_APP_STORAGE_BACKBLAZE_ACCESS_KEY', '');
                 $backblazeSecretKey = System::getEnv('_APP_STORAGE_BACKBLAZE_SECRET', '');
@@ -1653,3 +1698,6 @@ App::setResource('requestTimestamp', function ($request) {
     }
     return $requestTimestamp;
 }, ['request']);
+App::setResource('plan', function (array $plan = []) {
+    return [];
+});

app/realtime.php

@@ -26,6 +26,7 @@ use Utopia\Database\Helpers\ID;
 use Utopia\Database\Helpers\Role;
 use Utopia\Database\Query;
 use Utopia\Database\Validator\Authorization;
+use Utopia\DSN\DSN;
 use Utopia\Logger\Log;
 use Utopia\System\System;
 use Utopia\WebSocket\Adapter;
@@ -38,74 +39,100 @@ require_once __DIR__ . '/init.php';
 
 Runtime::enableCoroutine(SWOOLE_HOOK_ALL);
 
-function getConsoleDB(): Database
-{
-    global $register;
+// Allows overriding
+if (!function_exists("getConsoleDB")) {
+    function getConsoleDB(): Database
+    {
+        global $register;
 
-    /** @var \Utopia\Pools\Group $pools */
-    $pools = $register->get('pools');
+        /** @var \Utopia\Pools\Group $pools */
+        $pools = $register->get('pools');
 
-    $dbAdapter = $pools
-        ->get('console')
-        ->pop()
-        ->getResource()
-    ;
-
-    $database = new Database($dbAdapter, getCache());
-
-    $database
-        ->setNamespace('_console')
-        ->setMetadata('host', \gethostname())
-        ->setMetadata('project', '_console');
-
-    return $database;
-}
-
-function getProjectDB(Document $project): Database
-{
-    global $register;
-
-    /** @var \Utopia\Pools\Group $pools */
-    $pools = $register->get('pools');
-
-    if ($project->isEmpty() || $project->getId() === 'console') {
-        return getConsoleDB();
-    }
-
-    $dbAdapter = $pools
-        ->get($project->getAttribute('database'))
-        ->pop()
-        ->getResource()
-    ;
-
-    $database = new Database($dbAdapter, getCache());
-
-    $database
-        ->setNamespace('_' . $project->getInternalId())
-        ->setMetadata('host', \gethostname())
-        ->setMetadata('project', $project->getId());
-
-    return $database;
-}
-
-function getCache(): Cache
-{
-    global $register;
-
-    $pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */
-
-    $list = Config::getParam('pools-cache', []);
-    $adapters = [];
-
-    foreach ($list as $value) {
-        $adapters[] = $pools
-            ->get($value)
+        $dbAdapter = $pools
+            ->get('console')
             ->pop()
             ->getResource()
         ;
-    }
 
-    return new Cache(new Sharding($adapters));
+        $database = new Database($dbAdapter, getCache());
+
+        $database
+            ->setNamespace('_console')
+            ->setMetadata('host', \gethostname())
+            ->setMetadata('project', '_console');
+
+        return $database;
+    }
+}
+
+// Allows overriding
+if (!function_exists("getProjectDB")) {
+    function getProjectDB(Document $project): Database
+    {
+        global $register;
+
+        /** @var \Utopia\Pools\Group $pools */
+        $pools = $register->get('pools');
+
+        if ($project->isEmpty() || $project->getId() === 'console') {
+            return getConsoleDB();
+        }
+
+        try {
+            $dsn = new DSN($project->getAttribute('database'));
+        } catch (\InvalidArgumentException) {
+            // TODO: Temporary until all projects are using shared tables
+            $dsn = new DSN('mysql://' . $project->getAttribute('database'));
+        }
+
+        $adapter = $pools
+            ->get($dsn->getHost())
+            ->pop()
+            ->getResource();
+
+        $database = new Database($adapter, getCache());
+
+        if ($dsn->getHost() === DATABASE_SHARED_TABLES) {
+            $database
+                ->setSharedTables(true)
+                ->setTenant($project->getInternalId())
+                ->setNamespace($dsn->getParam('namespace'));
+        } else {
+            $database
+                ->setSharedTables(false)
+                ->setTenant(null)
+                ->setNamespace('_' . $project->getInternalId());
+        }
+
+        $database
+            ->setMetadata('host', \gethostname())
+            ->setMetadata('project', $project->getId());
+
+        return $database;
+    }
+}
+
+// Allows overriding
+if (!function_exists("getCache")) {
+    function getCache(): Cache
+    {
+        global $register;
+
+        $pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */
+
+        $list = Config::getParam('pools-cache', []);
+        $adapters = [];
+
+        foreach ($list as $value) {
+            $adapters[] = $pools
+                ->get($value)
+                ->pop()
+                ->getResource()
+            ;
+        }
+
+        return new Cache(new Sharding($adapters));
+    }
 }
 
 $realtime = new Realtime();
@@ -206,29 +233,32 @@ $server->onStart(function () use ($stats, $register, $containerId, &$statsDocume
     /**
      * Save current connections to the Database every 5 seconds.
      */
-    Timer::tick(5000, function () use ($register, $stats, &$statsDocument, $logError) {
-        $payload = [];
-        foreach ($stats as $projectId => $value) {
-            $payload[$projectId] = $stats->get($projectId, 'connectionsTotal');
-        }
-        if (empty($payload) || empty($statsDocument)) {
-            return;
-        }
+    // TODO: Remove this if check once it doesn't cause issues for cloud
+    if (System::getEnv('_APP_EDITION', 'self-hosted') === 'self-hosted') {
+        Timer::tick(5000, function () use ($register, $stats, &$statsDocument, $logError) {
+            $payload = [];
+            foreach ($stats as $projectId => $value) {
+                $payload[$projectId] = $stats->get($projectId, 'connectionsTotal');
+            }
+            if (empty($payload) || empty($statsDocument)) {
+                return;
+            }
 
-        try {
-            $database = getConsoleDB();
+            try {
+                $database = getConsoleDB();
 
-            $statsDocument
-                ->setAttribute('timestamp', DateTime::now())
-                ->setAttribute('value', json_encode($payload));
+                $statsDocument
+                    ->setAttribute('timestamp', DateTime::now())
+                    ->setAttribute('value', json_encode($payload));
 
-            Authorization::skip(fn () => $database->updateDocument('realtime', $statsDocument->getId(), $statsDocument));
-        } catch (Throwable $th) {
-            call_user_func($logError, $th, "updateWorkerDocument");
-        } finally {
-            $register->get('pools')->reclaim();
-        }
-    });
+                Authorization::skip(fn () => $database->updateDocument('realtime', $statsDocument->getId(), $statsDocument));
+            } catch (Throwable $th) {
+                call_user_func($logError, $th, "updateWorkerDocument");
+            } finally {
+                $register->get('pools')->reclaim();
+            }
+        });
+    }
 });
 
 $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats, $realtime, $logError) {
@@ -241,53 +271,56 @@ $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats,
         /**
          * Sending current connections to project channels on the console project every 5 seconds.
          */
-        if ($realtime->hasSubscriber('console', Role::users()->toString(), 'project')) {
-            $database = getConsoleDB();
+        // TODO: Remove this if check once it doesn't cause issues for cloud
+        if (System::getEnv('_APP_EDITION', 'self-hosted') === 'self-hosted') {
+            if ($realtime->hasSubscriber('console', Role::users()->toString(), 'project')) {
+                $database = getConsoleDB();
 
-            $payload = [];
+                $payload = [];
 
-            $list = Authorization::skip(fn () => $database->find('realtime', [
-                Query::greaterThan('timestamp', DateTime::addSeconds(new \DateTime(), -15)),
-            ]));
+                $list = Authorization::skip(fn () => $database->find('realtime', [
+                    Query::greaterThan('timestamp', DateTime::addSeconds(new \DateTime(), -15)),
+                ]));
 
-            /**
-             * Aggregate stats across containers.
-             */
-            foreach ($list as $document) {
-                foreach (json_decode($document->getAttribute('value')) as $projectId => $value) {
-                    if (array_key_exists($projectId, $payload)) {
-                        $payload[$projectId] +=  $value;
-                    } else {
-                        $payload[$projectId] =  $value;
+                /**
+                 * Aggregate stats across containers.
+                 */
+                foreach ($list as $document) {
+                    foreach (json_decode($document->getAttribute('value')) as $projectId => $value) {
+                        if (array_key_exists($projectId, $payload)) {
+                            $payload[$projectId] += $value;
+                        } else {
+                            $payload[$projectId] = $value;
+                        }
                     }
                 }
-            }
 
-            foreach ($stats as $projectId => $value) {
-                if (!array_key_exists($projectId, $payload)) {
-                    continue;
+                foreach ($stats as $projectId => $value) {
+                    if (!array_key_exists($projectId, $payload)) {
+                        continue;
+                    }
+
+                    $event = [
+                        'project' => 'console',
+                        'roles' => ['team:' . $stats->get($projectId, 'teamId')],
+                        'data' => [
+                            'events' => ['stats.connections'],
+                            'channels' => ['project'],
+                            'timestamp' => DateTime::formatTz(DateTime::now()),
+                            'payload' => [
+                                $projectId => $payload[$projectId]
+                            ]
+                        ]
+                    ];
+
+                    $server->send($realtime->getSubscribers($event), json_encode([
+                        'type' => 'event',
+                        'data' => $event['data']
+                    ]));
                 }
 
-                $event = [
-                    'project' => 'console',
-                    'roles' => ['team:' . $stats->get($projectId, 'teamId')],
-                    'data' => [
-                        'events' => ['stats.connections'],
-                        'channels' => ['project'],
-                        'timestamp' => DateTime::formatTz(DateTime::now()),
-                        'payload' => [
-                            $projectId => $payload[$projectId]
-                        ]
-                    ]
-                ];
-
-                $server->send($realtime->getSubscribers($event), json_encode([
-                    'type' => 'event',
-                    'data' => $event['data']
-                ]));
+                $register->get('pools')->reclaim();
             }
-
-            $register->get('pools')->reclaim();
         }
         /**
          * Sending test message for SDK E2E tests every 5 seconds.
@@ -484,16 +517,22 @@ $server->onOpen(function (int $connection, SwooleRequest $request) use ($server,
     } catch (Throwable $th) {
         call_user_func($logError, $th, "initServer");
 
+        // Handle SQL error code is 'HY000'
+        $code = $th->getCode();
+        if (!is_int($code)) {
+            $code = 500;
+        }
+
         $response = [
             'type' => 'error',
             'data' => [
-                'code' => $th->getCode(),
+                'code' => $code,
                 'message' => $th->getMessage()
             ]
         ];
 
         $server->send([$connection], json_encode($response));
-        $server->close($connection, $th->getCode());
+        $server->close($connection, $code);
 
         if (App::isDevelopment()) {
             Console::error('[Error] Connection Error');

app/views/install/compose.phtml

@@ -11,9 +11,7 @@ $httpsPort = $this->getParam('httpsPort', '');
 $version = $this->getParam('version', '');
 $organization = $this->getParam('organization', '');
 $image = $this->getParam('image', '');
-?>version: '3'
-
-services:
+?>services:
   traefik:
     image: traefik:2.11
     container_name: appwrite-traefik
@@ -750,7 +748,7 @@ services:
 
   openruntimes-executor:
     container_name: openruntimes-executor
-    hostname: appwrite-executor
+    hostname: exc1
     <<: *x-logging
     restart: unless-stopped
     stop_signal: SIGINT

app/worker.php

@@ -16,6 +16,7 @@ use Appwrite\Event\Usage;
 use Appwrite\Event\UsageDump;
 use Appwrite\Platform\Appwrite;
 use Swoole\Runtime;
+use Utopia\App;
 use Utopia\Cache\Adapter\Sharding;
 use Utopia\Cache\Cache;
 use Utopia\CLI\Console;
@@ -24,6 +25,7 @@ use Utopia\Database\Database;
 use Utopia\Database\DateTime;
 use Utopia\Database\Document;
 use Utopia\Database\Validator\Authorization;
+use Utopia\DSN\DSN;
 use Utopia\Logger\Log;
 use Utopia\Logger\Logger;
 use Utopia\Platform\Service;
@@ -69,14 +71,41 @@ Server::setResource('dbForProject', function (Cache $cache, Registry $register,
     }
 
     $pools = $register->get('pools');
-    $database = $pools
-        ->get($project->getAttribute('database'))
+
+    try {
+        $dsn = new DSN($project->getAttribute('database'));
+    } catch (\InvalidArgumentException) {
+        // TODO: Temporary until all projects are using shared tables
+        $dsn = new DSN('mysql://' . $project->getAttribute('database'));
+    }
+
+    $adapter = $pools
+        ->get($dsn->getHost())
         ->pop()
         ->getResource();
 
-    $adapter = new Database($database, $cache);
-    $adapter->setNamespace('_' . $project->getInternalId());
-    return $adapter;
+    $database = new Database($adapter, $cache);
+
+    try {
+        $dsn = new DSN($project->getAttribute('database'));
+    } catch (\InvalidArgumentException) {
+        // TODO: Temporary until all projects are using shared tables
+        $dsn = new DSN('mysql://' . $project->getAttribute('database'));
+    }
+
+    if ($dsn->getHost() === DATABASE_SHARED_TABLES) {
+        $database
+            ->setSharedTables(true)
+            ->setTenant($project->getInternalId())
+            ->setNamespace($dsn->getParam('namespace'));
+    } else {
+        $database
+            ->setSharedTables(false)
+            ->setTenant(null)
+            ->setNamespace('_' . $project->getInternalId());
+    }
+
+    return $database;
 }, ['cache', 'register', 'message', 'project', 'dbForConsole']);
 
 Server::setResource('getProjectDB', function (Group $pools, Database $dbForConsole, $cache) {
@@ -87,24 +116,51 @@ Server::setResource('getProjectDB', function (Group $pools, Database $dbForConso
             return $dbForConsole;
         }
 
-        $databaseName = $project->getAttribute('database');
+        try {
+            $dsn = new DSN($project->getAttribute('database'));
+        } catch (\InvalidArgumentException) {
+            // TODO: Temporary until all projects are using shared tables
+            $dsn = new DSN('mysql://' . $project->getAttribute('database'));
+        }
+
+        if (isset($databases[$dsn->getHost()])) {
+            $database = $databases[$dsn->getHost()];
+
+            if ($dsn->getHost() === DATABASE_SHARED_TABLES) {
+                $database
+                    ->setSharedTables(true)
+                    ->setTenant($project->getInternalId())
+                    ->setNamespace($dsn->getParam('namespace'));
+            } else {
+                $database
+                    ->setSharedTables(false)
+                    ->setTenant(null)
+                    ->setNamespace('_' . $project->getInternalId());
+            }
 
-        if (isset($databases[$databaseName])) {
-            $database = $databases[$databaseName];
-            $database->setNamespace('_' . $project->getInternalId());
             return $database;
         }
 
         $dbAdapter = $pools
-            ->get($databaseName)
+            ->get($dsn->getHost())
             ->pop()
             ->getResource();
 
         $database = new Database($dbAdapter, $cache);
 
-        $databases[$databaseName] = $database;
+        $databases[$dsn->getHost()] = $database;
 
-        $database->setNamespace('_' . $project->getInternalId());
+        if ($dsn->getHost() === DATABASE_SHARED_TABLES) {
+            $database
+                ->setSharedTables(true)
+                ->setTenant($project->getInternalId())
+                ->setNamespace($dsn->getParam('namespace'));
+        } else {
+            $database
+                ->setSharedTables(false)
+                ->setTenant(null)
+                ->setNamespace('_' . $project->getInternalId());
+        }
 
         return $database;
     };
@@ -245,7 +301,7 @@ try {
      * Any worker can be configured with the following env vars:
      * - _APP_WORKERS_NUM           The total number of worker processes
      * - _APP_WORKER_PER_CORE       The number of worker processes per core (ignored if _APP_WORKERS_NUM is set)
-     * - _APP_QUEUE_NAME  The name of the queue to read for database events
+     * - _APP_QUEUE_NAME            The name of the queue to read for database events
      */
     $platform->init(Service::TYPE_WORKER, [
         'workersNum' => System::getEnv('_APP_WORKERS_NUM', 1),
@@ -277,10 +333,6 @@ $worker
         $pools->reclaim();
         $version = System::getEnv('_APP_VERSION', 'UNKNOWN');
 
-        if ($error instanceof PDOException) {
-            throw $error;
-        }
-
         if ($logger) {
             $log->setNamespace("appwrite-worker");
             $log->setServer(\gethostname());

composer.json

@@ -52,16 +52,18 @@
         "utopia-php/config": "0.2.*",
         "utopia-php/database": "0.49.*",
         "utopia-php/domains": "0.5.*",
-        "utopia-php/dsn": "0.2.*",
+        "utopia-php/dsn": "0.2.1",
         "utopia-php/framework": "0.33.*",
+        "utopia-php/fetch": "0.2.*",
         "utopia-php/image": "0.6.*",
         "utopia-php/locale": "0.4.*",
-        "utopia-php/logger": "0.3.*",
-        "utopia-php/messaging": "0.10.*",
+        "utopia-php/logger": "0.5.*",
+        "utopia-php/messaging": "0.11.*",
         "utopia-php/migration": "0.4.*",
         "utopia-php/orchestration": "0.9.*",
         "utopia-php/platform": "0.5.*",
-        "utopia-php/pools": "0.4.*",
+        "utopia-php/pools": "0.5.*",
+        "utopia-php/preloader": "0.2.*",
         "utopia-php/queue": "0.7.*",
         "utopia-php/registry": "0.5.*",
         "utopia-php/storage": "0.18.*",
@@ -82,9 +84,8 @@
         "ext-fileinfo": "*",
         "appwrite/sdk-generator": "0.38.*",
         "phpunit/phpunit": "9.5.20",
-        "swoole/ide-helper": "5.0.2",
+        "swoole/ide-helper": "5.1.2",
         "textalk/websocket": "1.5.7",
-        "utopia-php/fetch": "0.1.*",
         "laravel/pint": "^1.14"
     },
     "provide": {

composer.lock

@@ -4,7 +4,7 @@
         "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
         "This file is @generated automatically"
     ],
-    "content-hash": "ddce40668e20709ace22219623303e38",
+    "content-hash": "53996479cd4ba0c73dbc72d46b240be0",
     "packages": [
         {
             "name": "adhocore/jwt",
@@ -480,6 +480,89 @@
             ],
             "time": "2022-09-10T18:51:20+00:00"
         },
+        {
+            "name": "giggsey/libphonenumber-for-php-lite",
+            "version": "8.13.36",
+            "source": {
+                "type": "git",
+                "url": "https://github.com/giggsey/libphonenumber-for-php-lite.git",
+                "reference": "144bbe70d67664b5245910a475c7190ff140ab4b"
+            },
+            "dist": {
+                "type": "zip",
+                "url": "https://api.github.com/repos/giggsey/libphonenumber-for-php-lite/zipball/144bbe70d67664b5245910a475c7190ff140ab4b",
+                "reference": "144bbe70d67664b5245910a475c7190ff140ab4b",
+                "shasum": ""
+            },
+            "require": {
+                "php": "^8.1",
+                "symfony/polyfill-mbstring": "^1.17"
+            },
+            "conflict": {
+                "giggsey/libphonenumber-for-php": "*"
+            },
+            "require-dev": {
+                "ext-dom": "*",
+                "friendsofphp/php-cs-fixer": "^3.12",
+                "infection/infection": "^0.28",
+                "pear/pear-core-minimal": "^1.10.11",
+                "pear/pear_exception": "^1.0.2",
+                "pear/versioncontrol_git": "^0.7",
+                "phing/phing": "^2.17.4",
+                "phpstan/extension-installer": "^1.2",
+                "phpstan/phpstan": "^1.8",
+                "phpstan/phpstan-phpunit": "^1.2",
+                "phpunit/phpunit": "^10.5",
+                "symfony/console": "^6.0",
+                "symfony/var-exporter": "^6.0"
+            },
+            "suggest": {
+                "giggsey/libphonenumber-for-php": "Use libphonenumber-for-php for geocoding, carriers, timezones and matching"
+            },
+            "type": "library",
+            "extra": {
+                "branch-alias": {
+                    "dev-master": "8.x-dev"
+                }
+            },
+            "autoload": {
+                "psr-4": {
+                    "libphonenumber\\": "src/"
+                },
+                "exclude-from-classmap": [
+                    "/src/data/",
+                    "/src/carrier/data/",
+                    "/src/geocoding/data/",
+                    "/src/timezone/data/"
+                ]
+            },
+            "notification-url": "https://packagist.org/downloads/",
+            "license": [
+                "Apache-2.0"
+            ],
+            "authors": [
+                {
+                    "name": "Joshua Gigg",
+                    "email": "giggsey@gmail.com",
+                    "homepage": "https://giggsey.com/"
+                }
+            ],
+            "description": "A lite version of giggsey/libphonenumber-for-php, which is a PHP Port of Google's libphonenumber",
+            "homepage": "https://github.com/giggsey/libphonenumber-for-php-lite",
+            "keywords": [
+                "geocoding",
+                "geolocation",
+                "libphonenumber",
+                "mobile",
+                "phonenumber",
+                "validation"
+            ],
+            "support": {
+                "issues": "https://github.com/giggsey/libphonenumber-for-php-lite/issues",
+                "source": "https://github.com/giggsey/libphonenumber-for-php-lite"
+            },
+            "time": "2024-05-03T06:31:11+00:00"
+        },
         {
             "name": "jean85/pretty-package-versions",
             "version": "2.0.6",
@@ -1043,6 +1126,86 @@
             },
             "time": "2022-03-17T08:00:35+00:00"
         },
+        {
+            "name": "symfony/polyfill-mbstring",
+            "version": "v1.29.0",
+            "source": {
+                "type": "git",
+                "url": "https://github.com/symfony/polyfill-mbstring.git",
+                "reference": "9773676c8a1bb1f8d4340a62efe641cf76eda7ec"
+            },
+            "dist": {
+                "type": "zip",
+                "url": "https://api.github.com/repos/symfony/polyfill-mbstring/zipball/9773676c8a1bb1f8d4340a62efe641cf76eda7ec",
+                "reference": "9773676c8a1bb1f8d4340a62efe641cf76eda7ec",
+                "shasum": ""
+            },
+            "require": {
+                "php": ">=7.1"
+            },
+            "provide": {
+                "ext-mbstring": "*"
+            },
+            "suggest": {
+                "ext-mbstring": "For best performance"
+            },
+            "type": "library",
+            "extra": {
+                "thanks": {
+                    "name": "symfony/polyfill",
+                    "url": "https://github.com/symfony/polyfill"
+                }
+            },
+            "autoload": {
+                "files": [
+                    "bootstrap.php"
+                ],
+                "psr-4": {
+                    "Symfony\\Polyfill\\Mbstring\\": ""
+                }
+            },
+            "notification-url": "https://packagist.org/downloads/",
+            "license": [
+                "MIT"
+            ],
+            "authors": [
+                {
+                    "name": "Nicolas Grekas",
+                    "email": "p@tchwork.com"
+                },
+                {
+                    "name": "Symfony Community",
+                    "homepage": "https://symfony.com/contributors"
+                }
+            ],
+            "description": "Symfony polyfill for the Mbstring extension",
+            "homepage": "https://symfony.com",
+            "keywords": [
+                "compatibility",
+                "mbstring",
+                "polyfill",
+                "portable",
+                "shim"
+            ],
+            "support": {
+                "source": "https://github.com/symfony/polyfill-mbstring/tree/v1.29.0"
+            },
+            "funding": [
+                {
+                    "url": "https://symfony.com/sponsor",
+                    "type": "custom"
+                },
+                {
+                    "url": "https://github.com/fabpot",
+                    "type": "github"
+                },
+                {
+                    "url": "https://tidelift.com/funding/github/packagist/symfony/symfony",
+                    "type": "tidelift"
+                }
+            ],
+            "time": "2024-01-29T20:11:03+00:00"
+        },
         {
             "name": "symfony/polyfill-php80",
             "version": "v1.29.0",
@@ -1717,6 +1880,45 @@
             },
             "time": "2024-05-07T02:01:25+00:00"
         },
+        {
+            "name": "utopia-php/fetch",
+            "version": "0.2.1",
+            "source": {
+                "type": "git",
+                "url": "https://github.com/utopia-php/fetch.git",
+                "reference": "1423c0ee3eef944d816ca6e31706895b585aea82"
+            },
+            "dist": {
+                "type": "zip",
+                "url": "https://api.github.com/repos/utopia-php/fetch/zipball/1423c0ee3eef944d816ca6e31706895b585aea82",
+                "reference": "1423c0ee3eef944d816ca6e31706895b585aea82",
+                "shasum": ""
+            },
+            "require": {
+                "php": ">=8.0"
+            },
+            "require-dev": {
+                "laravel/pint": "^1.5.0",
+                "phpstan/phpstan": "^1.10",
+                "phpunit/phpunit": "^9.5"
+            },
+            "type": "library",
+            "autoload": {
+                "psr-4": {
+                    "Utopia\\Fetch\\": "src/"
+                }
+            },
+            "notification-url": "https://packagist.org/downloads/",
+            "license": [
+                "MIT"
+            ],
+            "description": "A simple library that provides an interface for making HTTP Requests.",
+            "support": {
+                "issues": "https://github.com/utopia-php/fetch/issues",
+                "source": "https://github.com/utopia-php/fetch/tree/0.2.1"
+            },
+            "time": "2024-03-18T11:50:59+00:00"
+        },
         {
             "name": "utopia-php/framework",
             "version": "0.33.6",
@@ -1863,22 +2065,23 @@
         },
         {
             "name": "utopia-php/logger",
-            "version": "0.3.2",
+            "version": "0.5.2",
             "source": {
                 "type": "git",
                 "url": "https://github.com/utopia-php/logger.git",
-                "reference": "ba763c10688fe2ed715ad2bed3f13d18dfec6253"
+                "reference": "c6dfdb672e41364c309b0c30dc03bc6d45446dba"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/utopia-php/logger/zipball/ba763c10688fe2ed715ad2bed3f13d18dfec6253",
-                "reference": "ba763c10688fe2ed715ad2bed3f13d18dfec6253",
+                "url": "https://api.github.com/repos/utopia-php/logger/zipball/c6dfdb672e41364c309b0c30dc03bc6d45446dba",
+                "reference": "c6dfdb672e41364c309b0c30dc03bc6d45446dba",
                 "shasum": ""
             },
             "require": {
                 "php": ">=8.0"
             },
             "require-dev": {
+                "laravel/pint": "1.2.*",
                 "phpstan/phpstan": "1.9.x-dev",
                 "phpunit/phpunit": "^9.3",
                 "vimeo/psalm": "4.0.1"
@@ -1910,27 +2113,28 @@
             ],
             "support": {
                 "issues": "https://github.com/utopia-php/logger/issues",
-                "source": "https://github.com/utopia-php/logger/tree/0.3.2"
+                "source": "https://github.com/utopia-php/logger/tree/0.5.2"
             },
-            "time": "2023-11-22T14:45:43+00:00"
+            "time": "2024-05-17T09:32:59+00:00"
         },
         {
             "name": "utopia-php/messaging",
-            "version": "0.10.0",
+            "version": "0.11.0",
             "source": {
                 "type": "git",
                 "url": "https://github.com/utopia-php/messaging.git",
-                "reference": "71dce00ad43eb278a877cb2c329f7b8d677adfeb"
+                "reference": "b499c3ad11af711c28252c62d83f24e6106a2154"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/utopia-php/messaging/zipball/71dce00ad43eb278a877cb2c329f7b8d677adfeb",
-                "reference": "71dce00ad43eb278a877cb2c329f7b8d677adfeb",
+                "url": "https://api.github.com/repos/utopia-php/messaging/zipball/b499c3ad11af711c28252c62d83f24e6106a2154",
+                "reference": "b499c3ad11af711c28252c62d83f24e6106a2154",
                 "shasum": ""
             },
             "require": {
                 "ext-curl": "*",
                 "ext-openssl": "*",
+                "giggsey/libphonenumber-for-php-lite": "8.13.36",
                 "php": ">=8.0.0",
                 "phpmailer/phpmailer": "6.9.1"
             },
@@ -1960,9 +2164,9 @@
             ],
             "support": {
                 "issues": "https://github.com/utopia-php/messaging/issues",
-                "source": "https://github.com/utopia-php/messaging/tree/0.10.0"
+                "source": "https://github.com/utopia-php/messaging/tree/0.11.0"
             },
-            "time": "2024-02-20T07:30:15+00:00"
+            "time": "2024-05-08T17:10:02+00:00"
         },
         {
             "name": "utopia-php/migration",
@@ -2172,16 +2376,16 @@
         },
         {
             "name": "utopia-php/pools",
-            "version": "0.4.2",
+            "version": "0.5.0",
             "source": {
                 "type": "git",
                 "url": "https://github.com/utopia-php/pools.git",
-                "reference": "d2870ab74b31b7f4027799f082e85122154f8bed"
+                "reference": "6f716a213a08db95eda1b5dddfa90983c1834817"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/utopia-php/pools/zipball/d2870ab74b31b7f4027799f082e85122154f8bed",
-                "reference": "d2870ab74b31b7f4027799f082e85122154f8bed",
+                "url": "https://api.github.com/repos/utopia-php/pools/zipball/6f716a213a08db95eda1b5dddfa90983c1834817",
+                "reference": "6f716a213a08db95eda1b5dddfa90983c1834817",
                 "shasum": ""
             },
             "require": {
@@ -2217,9 +2421,62 @@
             ],
             "support": {
                 "issues": "https://github.com/utopia-php/pools/issues",
-                "source": "https://github.com/utopia-php/pools/tree/0.4.2"
+                "source": "https://github.com/utopia-php/pools/tree/0.5.0"
             },
-            "time": "2022-11-22T07:55:45+00:00"
+            "time": "2024-04-19T11:11:54+00:00"
+        },
+        {
+            "name": "utopia-php/preloader",
+            "version": "0.2.4",
+            "source": {
+                "type": "git",
+                "url": "https://github.com/utopia-php/preloader.git",
+                "reference": "65ef48392e72172f584b0baa2e224f9a1cebcce0"
+            },
+            "dist": {
+                "type": "zip",
+                "url": "https://api.github.com/repos/utopia-php/preloader/zipball/65ef48392e72172f584b0baa2e224f9a1cebcce0",
+                "reference": "65ef48392e72172f584b0baa2e224f9a1cebcce0",
+                "shasum": ""
+            },
+            "require": {
+                "php": ">=7.1"
+            },
+            "require-dev": {
+                "phpunit/phpunit": "^9.3",
+                "vimeo/psalm": "4.0.1"
+            },
+            "type": "library",
+            "autoload": {
+                "psr-4": {
+                    "Utopia\\Preloader\\": "src/Preloader"
+                }
+            },
+            "notification-url": "https://packagist.org/downloads/",
+            "license": [
+                "MIT"
+            ],
+            "authors": [
+                {
+                    "name": "Eldad Fux",
+                    "email": "team@appwrite.io"
+                }
+            ],
+            "description": "Utopia Preloader library is simple and lite library for managing PHP preloading configuration",
+            "keywords": [
+                "framework",
+                "php",
+                "preload",
+                "preloader",
+                "preloading",
+                "upf",
+                "utopia"
+            ],
+            "support": {
+                "issues": "https://github.com/utopia-php/preloader/issues",
+                "source": "https://github.com/utopia-php/preloader/tree/0.2.4"
+            },
+            "time": "2020-10-24T07:04:59+00:00"
         },
         {
             "name": "utopia-php/queue",
@@ -5049,16 +5306,16 @@
         },
         {
             "name": "swoole/ide-helper",
-            "version": "5.0.2",
+            "version": "5.1.2",
             "source": {
                 "type": "git",
                 "url": "https://github.com/swoole/ide-helper.git",
-                "reference": "16cfee44a6ec92254228c39bcab2fb8ae74cc2ea"
+                "reference": "33ec7af9111b76d06a70dd31191cc74793551112"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/swoole/ide-helper/zipball/16cfee44a6ec92254228c39bcab2fb8ae74cc2ea",
-                "reference": "16cfee44a6ec92254228c39bcab2fb8ae74cc2ea",
+                "url": "https://api.github.com/repos/swoole/ide-helper/zipball/33ec7af9111b76d06a70dd31191cc74793551112",
+                "reference": "33ec7af9111b76d06a70dd31191cc74793551112",
                 "shasum": ""
             },
             "type": "library",
@@ -5075,9 +5332,9 @@
             "description": "IDE help files for Swoole.",
             "support": {
                 "issues": "https://github.com/swoole/ide-helper/issues",
-                "source": "https://github.com/swoole/ide-helper/tree/5.0.2"
+                "source": "https://github.com/swoole/ide-helper/tree/5.1.2"
             },
-            "time": "2023-03-20T06:05:55+00:00"
+            "time": "2024-02-01T22:28:11+00:00"
         },
         {
             "name": "symfony/polyfill-ctype",
@@ -5158,86 +5415,6 @@
             ],
             "time": "2024-01-29T20:11:03+00:00"
         },
-        {
-            "name": "symfony/polyfill-mbstring",
-            "version": "v1.29.0",
-            "source": {
-                "type": "git",
-                "url": "https://github.com/symfony/polyfill-mbstring.git",
-                "reference": "9773676c8a1bb1f8d4340a62efe641cf76eda7ec"
-            },
-            "dist": {
-                "type": "zip",
-                "url": "https://api.github.com/repos/symfony/polyfill-mbstring/zipball/9773676c8a1bb1f8d4340a62efe641cf76eda7ec",
-                "reference": "9773676c8a1bb1f8d4340a62efe641cf76eda7ec",
-                "shasum": ""
-            },
-            "require": {
-                "php": ">=7.1"
-            },
-            "provide": {
-                "ext-mbstring": "*"
-            },
-            "suggest": {
-                "ext-mbstring": "For best performance"
-            },
-            "type": "library",
-            "extra": {
-                "thanks": {
-                    "name": "symfony/polyfill",
-                    "url": "https://github.com/symfony/polyfill"
-                }
-            },
-            "autoload": {
-                "files": [
-                    "bootstrap.php"
-                ],
-                "psr-4": {
-                    "Symfony\\Polyfill\\Mbstring\\": ""
-                }
-            },
-            "notification-url": "https://packagist.org/downloads/",
-            "license": [
-                "MIT"
-            ],
-            "authors": [
-                {
-                    "name": "Nicolas Grekas",
-                    "email": "p@tchwork.com"
-                },
-                {
-                    "name": "Symfony Community",
-                    "homepage": "https://symfony.com/contributors"
-                }
-            ],
-            "description": "Symfony polyfill for the Mbstring extension",
-            "homepage": "https://symfony.com",
-            "keywords": [
-                "compatibility",
-                "mbstring",
-                "polyfill",
-                "portable",
-                "shim"
-            ],
-            "support": {
-                "source": "https://github.com/symfony/polyfill-mbstring/tree/v1.29.0"
-            },
-            "funding": [
-                {
-                    "url": "https://symfony.com/sponsor",
-                    "type": "custom"
-                },
-                {
-                    "url": "https://github.com/fabpot",
-                    "type": "github"
-                },
-                {
-                    "url": "https://tidelift.com/funding/github/packagist/symfony/symfony",
-                    "type": "tidelift"
-                }
-            ],
-            "time": "2024-01-29T20:11:03+00:00"
-        },
         {
             "name": "textalk/websocket",
             "version": "1.5.7",
@@ -5408,45 +5585,6 @@
                 }
             ],
             "time": "2023-11-21T18:54:41+00:00"
-        },
-        {
-            "name": "utopia-php/fetch",
-            "version": "0.1.0",
-            "source": {
-                "type": "git",
-                "url": "https://github.com/utopia-php/fetch.git",
-                "reference": "2fa214b9262acd1a3583515a364da4f35929d5c5"
-            },
-            "dist": {
-                "type": "zip",
-                "url": "https://api.github.com/repos/utopia-php/fetch/zipball/2fa214b9262acd1a3583515a364da4f35929d5c5",
-                "reference": "2fa214b9262acd1a3583515a364da4f35929d5c5",
-                "shasum": ""
-            },
-            "require": {
-                "php": ">=8.0"
-            },
-            "require-dev": {
-                "laravel/pint": "^1.5.0",
-                "phpstan/phpstan": "^1.10",
-                "phpunit/phpunit": "^9.5"
-            },
-            "type": "library",
-            "autoload": {
-                "psr-4": {
-                    "Utopia\\Fetch\\": "src/"
-                }
-            },
-            "notification-url": "https://packagist.org/downloads/",
-            "license": [
-                "MIT"
-            ],
-            "description": "A simple library that provides an interface for making HTTP Requests.",
-            "support": {
-                "issues": "https://github.com/utopia-php/fetch/issues",
-                "source": "https://github.com/utopia-php/fetch/tree/0.1.0"
-            },
-            "time": "2023-10-10T11:58:32+00:00"
         }
     ],
     "aliases": [],

docker-compose.yml

@@ -10,8 +10,6 @@ x-logging: &x-logging
       max-file: "5"
       max-size: "10m"
 
-version: "3"
-
 services:
   traefik:
     image: traefik:2.11
@@ -53,7 +51,7 @@ services:
         DEBUG: false
         TESTING: true
         VERSION: dev
-    ports:
+    ports: 
       - 9501:80
     networks:
       - appwrite
@@ -94,6 +92,7 @@ services:
       - app/http.php
     environment:
       - _APP_ENV
+      - _APP_EDITION
       - _APP_WORKER_PER_CORE
       - _APP_LOCALE
       - _APP_CONSOLE_WHITELIST_ROOT
@@ -188,6 +187,8 @@ services:
       - _APP_MESSAGE_EMAIL_TEST_DSN
       - _APP_MESSAGE_PUSH_TEST_DSN
       - _APP_CONSOLE_COUNTRIES_DENYLIST
+      - _APP_EXPERIMENT_LOGGING_PROVIDER
+      - _APP_EXPERIMENT_LOGGING_CONFIG
 
   appwrite-realtime:
     entrypoint: realtime
@@ -812,7 +813,7 @@ services:
 
   openruntimes-executor:
     container_name: openruntimes-executor
-    hostname: appwrite-executor
+    hostname: exc1
     <<: *x-logging
     stop_signal: SIGINT
     image: openruntimes/executor:0.5.5
@@ -878,7 +879,7 @@ services:
       - OPR_PROXY_LOGGING_PROVIDER=$_APP_LOGGING_PROVIDER
       - OPR_PROXY_LOGGING_CONFIG=$_APP_LOGGING_CONFIG
       - OPR_PROXY_ALGORITHM=random
-      - OPR_PROXY_EXECUTORS=appwrite-executor
+      - OPR_PROXY_EXECUTORS=exc1
       - OPR_PROXY_HEALTHCHECK_INTERVAL=10000
       - OPR_PROXY_MAX_TIMEOUT=600
       - OPR_PROXY_HEALTHCHECK=enabled

src/Appwrite/Auth/Hash/Phpass.php

@@ -263,7 +263,7 @@ class Phpass extends Hash
          */
         $itoa64 = './ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789';
         $output = '$2a$';
-        $output .= chr(ord('0') + $options['iteration_count_log2'] / 10);
+        $output .= chr(ord('0') + intval($options['iteration_count_log2'] / 10));
         $output .= chr(ord('0') + $options['iteration_count_log2'] % 10);
         $output .= '$';
         $i = 0;

src/Appwrite/Docker/Compose.php

@@ -27,14 +27,6 @@ class Compose
         }
     }
 
-    /**
-     * @return string
-     */
-    public function getVersion(): string
-    {
-        return (isset($this->compose['version'])) ? $this->compose['version'] : '';
-    }
-
     /**
      * @return Service[]
      */

src/Appwrite/Event/Database.php

@@ -3,6 +3,7 @@
 namespace Appwrite\Event;
 
 use Utopia\Database\Document;
+use Utopia\DSN\DSN;
 use Utopia\Queue\Client;
 use Utopia\Queue\Connection;
 
@@ -107,18 +108,30 @@ class Database extends Event
      */
     public function trigger(): string|bool
     {
-        $this->setQueue($this->getProject()->getAttribute('database'));
+        try {
+            $dsn = new DSN($this->getProject()->getAttribute('database'));
+        } catch (\InvalidArgumentException) {
+            // TODO: Temporary until all projects are using shared tables
+            $dsn = new DSN('mysql://' . $this->getProject()->getAttribute('database'));
+        }
+
+        $this->setQueue($dsn->getHost());
 
         $client = new Client($this->queue, $this->connection);
 
-        return $client->enqueue([
-            'project' => $this->project,
-            'user' => $this->user,
-            'type' => $this->type,
-            'collection' => $this->collection,
-            'document' => $this->document,
-            'database' => $this->database,
-            'events' => Event::generateEvents($this->getEvent(), $this->getParams())
-        ]);
+        try {
+            $result = $client->enqueue([
+                'project' => $this->project,
+                'user' => $this->user,
+                'type' => $this->type,
+                'collection' => $this->collection,
+                'document' => $this->document,
+                'database' => $this->database,
+                'events' => Event::generateEvents($this->getEvent(), $this->getParams())
+            ]);
+            return $result;
+        } catch (\Throwable $th) {
+            return false;
+        }
     }
 }

src/Appwrite/Event/Event.php

@@ -49,6 +49,7 @@ class Event
     protected string $class = '';
     protected string $event = '';
     protected array $params = [];
+    protected array $sensitive = [];
     protected array $payload = [];
     protected array $context = [];
     protected ?Document $project = null;
@@ -158,12 +159,17 @@ class Event
      * Set payload for this event.
      *
      * @param array $payload
+     * @param array $sensitive
      * @return self
      */
-    public function setPayload(array $payload): self
+    public function setPayload(array $payload, array $sensitive = []): self
     {
         $this->payload = $payload;
 
+        foreach ($sensitive as $key) {
+            $this->sensitive[$key] = true;
+        }
+
         return $this;
     }
 
@@ -177,6 +183,19 @@ class Event
         return $this->payload;
     }
 
+    public function getRealtimePayload(): array
+    {
+        $payload = [];
+
+        foreach ($this->payload as $key => $value) {
+            if (!isset($this->sensitive[$key])) {
+                $payload[$key] = $value;
+            }
+        }
+
+        return $payload;
+    }
+
     /**
      * Set context for this event.
      *
@@ -239,6 +258,13 @@ class Event
         return $this;
     }
 
+    public function setParamSensitive(string $key): self
+    {
+        $this->sensitive[$key] = true;
+
+        return $this;
+    }
+
     /**
      * Get param of event.
      *
@@ -291,6 +317,7 @@ class Event
     public function reset(): self
     {
         $this->params = [];
+        $this->sensitive = [];
 
         return $this;
     }

src/Appwrite/Event/Usage.php

@@ -58,7 +58,6 @@ class Usage extends Event
     public function trigger(): string|bool
     {
         $client = new Client($this->queue, $this->connection);
-
         return $client->enqueue([
             'project' => $this->getProject(),
             'reduce'  => $this->reduce,

src/Appwrite/Event/Validator/Event.php

@@ -34,7 +34,7 @@ class Event extends Validator
     public function isValid($value): bool
     {
         $events = Config::getParam('events', []);
-        $parts = \explode('.', $value);
+        $parts = \explode('.', $value ?? '');
         $count = \count($parts);
 
         if ($count < 2 || $count > 7) {

src/Appwrite/Event/Validator/FunctionEvent.php

@@ -13,7 +13,7 @@ class FunctionEvent extends Event
      */
     public function isValid($value): bool
     {
-        if (str_starts_with($value, 'functions.')) {
+        if (str_starts_with($value ?? false, 'functions.')) {
             $this->message = 'Triggering a function on a function event is not allowed.';
             return false;
         }

src/Appwrite/Extend/Exception.php

@@ -196,6 +196,9 @@ class Exception extends \Exception
     public const ATTRIBUTE_VALUE_INVALID           = 'attribute_value_invalid';
     public const ATTRIBUTE_TYPE_INVALID            = 'attribute_type_invalid';
 
+    /** Relationship */
+    public const RELATIONSHIP_VALUE_INVALID        = 'relationship_value_invalid';
+
     /** Indexes */
     public const INDEX_NOT_FOUND                   = 'index_not_found';
     public const INDEX_LIMIT_EXCEEDED              = 'index_limit_exceeded';
@@ -298,11 +301,21 @@ class Exception extends \Exception
     protected array $errors = [];
     protected bool $publish;
 
-    public function __construct(string $type = Exception::GENERAL_UNKNOWN, string $message = null, int $code = null, \Throwable $previous = null)
+    public function __construct(string $type = Exception::GENERAL_UNKNOWN, string $message = null, int|string $code = null, \Throwable $previous = null)
     {
         $this->errors = Config::getParam('errors');
         $this->type = $type;
         $this->code = $code ?? $this->errors[$type]['code'];
+
+        // Mark string errors like HY001 from PDO as 500 errors
+        if(\is_string($this->code)) {
+            if (\is_numeric($this->code)) {
+                $this->code = (int) $this->code;
+            } else {
+                $this->code = 500;
+            }
+        }
+
         $this->message = $message ?? $this->errors[$type]['description'];
 
         $this->publish = $this->errors[$type]['publish'] ?? ($this->code >= 500);

src/Appwrite/Hooks/Hooks.php

@@ -16,11 +16,14 @@ class Hooks
 
     /**
      * @param mixed[] $params
+     * @return mixed
      */
-    public function trigger(string $name, array $params = [])
+    public function trigger(string $name, array $params = []): mixed
     {
         if (isset(self::$hooks[$name])) {
-            call_user_func_array(self::$hooks[$name], $params);
+            return call_user_func_array(self::$hooks[$name], $params);
         }
+
+        return null;
     }
 }

src/Appwrite/Migration/Migration.php

@@ -85,6 +85,7 @@ abstract class Migration
         '1.5.4'  => 'V20',
         '1.5.5'  => 'V20',
         '1.5.6'  => 'V20',
+        '1.5.7'  => 'V20',
     ];
 
     /**

src/Appwrite/Platform/Tasks/QueueCount.php

@@ -2,11 +2,11 @@
 
 namespace Appwrite\Platform\Tasks;
 
-use Appwrite\Event\Event;
 use Utopia\CLI\Console;
 use Utopia\Platform\Action;
 use Utopia\Queue\Client;
 use Utopia\Queue\Connection;
+use Utopia\Validator\Text;
 use Utopia\Validator\WhiteList;
 
 class QueueCount extends Action
@@ -21,19 +21,7 @@ class QueueCount extends Action
     {
         $this
             ->desc('Return the number of from a specific queue identified by the name parameter with a specific type')
-            ->param('name', '', new WhiteList([
-                Event::DATABASE_QUEUE_NAME,
-                Event::DELETE_QUEUE_NAME,
-                Event::AUDITS_QUEUE_NAME,
-                Event::MAILS_QUEUE_NAME,
-                Event::FUNCTIONS_QUEUE_NAME,
-                Event::USAGE_QUEUE_NAME,
-                Event::WEBHOOK_QUEUE_NAME,
-                Event::CERTIFICATES_QUEUE_NAME,
-                Event::BUILDS_QUEUE_NAME,
-                Event::MESSAGING_QUEUE_NAME,
-                Event::MIGRATIONS_QUEUE_NAME
-            ]), 'Queue name')
+            ->param('name', '', new Text(100), 'Queue name')
             ->param('type', '', new WhiteList([
                 'success',
                 'failed',

src/Appwrite/Platform/Tasks/QueueRetry.php

@@ -2,12 +2,11 @@
 
 namespace Appwrite\Platform\Tasks;
 
-use Appwrite\Event\Event;
 use Utopia\CLI\Console;
 use Utopia\Platform\Action;
 use Utopia\Queue\Client;
 use Utopia\Queue\Connection;
-use Utopia\Validator\WhiteList;
+use Utopia\Validator\Text;
 use Utopia\Validator\Wildcard;
 
 class QueueRetry extends Action
@@ -22,19 +21,7 @@ class QueueRetry extends Action
     {
         $this
             ->desc('Retry failed jobs from a specific queue identified by the name parameter')
-            ->param('name', '', new WhiteList([
-                Event::DATABASE_QUEUE_NAME,
-                Event::DELETE_QUEUE_NAME,
-                Event::AUDITS_QUEUE_NAME,
-                Event::MAILS_QUEUE_NAME,
-                Event::FUNCTIONS_QUEUE_NAME,
-                Event::USAGE_QUEUE_NAME,
-                Event::WEBHOOK_CLASS_NAME,
-                Event::CERTIFICATES_QUEUE_NAME,
-                Event::BUILDS_QUEUE_NAME,
-                Event::MESSAGING_QUEUE_NAME,
-                Event::MIGRATIONS_QUEUE_NAME
-            ]), 'Queue name')
+            ->param('name', '', new Text(100), 'Queue name')
             ->param('limit', 0, new Wildcard(), 'jobs limit', true)
             ->inject('queue')
             ->callback(fn ($name, $limit, $queue) => $this->action($name, $limit, $queue));

src/Appwrite/Platform/Workers/Builds.php

@@ -433,6 +433,8 @@ class Builds extends Action
                                         throw new \Exception('Build not found', 404);
                                     }
 
+                                    $logs = \mb_substr($logs, 0, null, 'UTF-8'); // Get only valid UTF8 part - removes leftover half-multibytes causing SQL errors
+
                                     $build = $build->setAttribute('logs', $build->getAttribute('logs', '') . $logs);
                                     $build = $dbForProject->updateDocument('builds', $build->getId(), $build);
 

src/Appwrite/Platform/Workers/Databases.php

@@ -116,6 +116,11 @@ class Databases extends Action
          */
         $attribute = $dbForProject->getDocument('attributes', $attribute->getId());
 
+        if ($attribute->isEmpty()) {
+            // Attribute was deleted before job was processed
+            return;
+        }
+
         $collectionId = $collection->getId();
         $key = $attribute->getAttribute('key', '');
         $type = $attribute->getAttribute('type', '');
@@ -176,7 +181,6 @@ class Databases extends Action
                 }
             }
 
-
             $dbForProject->updateDocument(
                 'attributes',
                 $attribute->getId(),

src/Appwrite/Platform/Workers/Deletes.php

@@ -12,6 +12,7 @@ use Utopia\Audit\Audit;
 use Utopia\Cache\Adapter\Filesystem;
 use Utopia\Cache\Cache;
 use Utopia\CLI\Console;
+use Utopia\Config\Config;
 use Utopia\Database\Database;
 use Utopia\Database\DateTime;
 use Utopia\Database\Document;
@@ -21,6 +22,7 @@ use Utopia\Database\Exception\Conflict;
 use Utopia\Database\Exception\Restricted;
 use Utopia\Database\Exception\Structure;
 use Utopia\Database\Query;
+use Utopia\DSN\DSN;
 use Utopia\Logger\Log;
 use Utopia\Platform\Action;
 use Utopia\Queue\Message;
@@ -441,7 +443,7 @@ class Deletes extends Action
      * @param Document $document
      * @return void
      * @throws Authorization
-     * @throws \Utopia\Database\Exception
+     * @throws DatabaseException
      * @throws Conflict
      * @throws Restricted
      * @throws Structure
@@ -469,25 +471,48 @@ class Deletes extends Action
      * @return void
      * @throws Exception
      * @throws Authorization
-     * @throws \Utopia\Database\Exception
+     * @throws DatabaseException
      */
     private function deleteProject(Database $dbForConsole, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, Document $document): void
     {
-        $projectId = $document->getId();
         $projectInternalId = $document->getInternalId();
 
-        // Delete project tables
+        try {
+            $dsn = new DSN($document->getAttribute('database', 'console'));
+        } catch (\InvalidArgumentException) {
+            // TODO: Temporary until all projects are using shared tables
+            $dsn = new DSN('mysql://' . $document->getAttribute('database', 'console'));
+        }
+
         $dbForProject = $getProjectDB($document);
 
-        while (true) {
-            $collections = $dbForProject->listCollections();
+        $projectCollectionIds = [
+            ...\array_keys(Config::getParam('collections', [])['projects']),
+            Audit::COLLECTION,
+            TimeLimit::COLLECTION,
+        ];
 
-            if (empty($collections)) {
-                break;
-            }
+        $limit = \count($projectCollectionIds) + 25;
+
+        while (true) {
+            $collections = $dbForProject->listCollections($limit);
 
             foreach ($collections as $collection) {
-                $dbForProject->deleteCollection($collection->getId());
+                if ($dsn->getHost() !== DATABASE_SHARED_TABLES || !\in_array($collection->getId(), $projectCollectionIds)) {
+                    $dbForProject->deleteCollection($collection->getId());
+                } else {
+                    $this->deleteByGroup($collection->getId(), [], database: $dbForProject);
+                }
+            }
+
+            if ($dsn->getHost() === DATABASE_SHARED_TABLES) {
+                $collectionsIds = \array_map(fn ($collection) => $collection->getId(), $collections);
+
+                if (empty(\array_diff($collectionsIds, $projectCollectionIds))) {
+                    break;
+                }
+            } elseif (empty($collections)) {
+                break;
             }
         }
 
@@ -523,17 +548,16 @@ class Deletes extends Action
             Query::equal('projectInternalId', [$projectInternalId]),
         ], $dbForConsole);
 
-        // Delete VCS commments
+        // Delete VCS comments
         $this->deleteByGroup('vcsComments', [
             Query::equal('projectInternalId', [$projectInternalId]),
         ], $dbForConsole);
 
-        // Delete metadata tables
-        try {
+        // Delete metadata table
+        if ($dsn->getHost() !== DATABASE_SHARED_TABLES) {
             $dbForProject->deleteCollection('_metadata');
-        } catch (\Throwable) {
-            // Ignore: deleteCollection tries to delete a metadata entry after the collection is deleted,
-            // which will throw an exception here because the metadata collection is already deleted.
+        } else {
+            $this->deleteByGroup('_metadata', [], $dbForProject);
         }
 
         // Delete all storage directories
@@ -660,9 +684,11 @@ class Deletes extends Action
         $dbForProject = $getProjectDB($project);
         $timeLimit = new TimeLimit("", 0, 1, $dbForProject);
         $abuse = new Abuse($timeLimit);
-        $status = $abuse->cleanup($abuseRetention);
-        if (!$status) {
-            throw new Exception('Failed to delete Abuse logs for project ' . $projectId);
+
+        try {
+            $abuse->cleanup($abuseRetention);
+        } catch (DatabaseException $e) {
+            Console::error('Failed to delete abuse logs for project ' . $projectId . ': ' . $e->getMessage());
         }
     }
 
@@ -678,9 +704,11 @@ class Deletes extends Action
         $projectId = $project->getId();
         $dbForProject = $getProjectDB($project);
         $audit = new Audit($dbForProject);
-        $status = $audit->cleanup($auditRetention);
-        if (!$status) {
-            throw new Exception('Failed to delete Audit logs for project' . $projectId);
+
+        try {
+            $audit->cleanup($auditRetention);
+        } catch (DatabaseException $e) {
+            Console::error('Failed to delete audit logs for project ' . $projectId . ': ' . $e->getMessage());
         }
     }
 
@@ -935,7 +963,12 @@ class Deletes extends Action
         while ($sum === $limit) {
             $chunk++;
 
-            $results = $database->find($collection, \array_merge([Query::limit($limit)], $queries));
+            try {
+                $results = $database->find($collection, [Query::limit($limit), ...$queries]);
+            } catch (DatabaseException $e) {
+                Console::error('Failed to find documents for collection ' . $collection . ': ' . $e->getMessage());
+                return;
+            }
 
             $sum = count($results);
 

src/Appwrite/Platform/Workers/Functions.php

@@ -198,6 +198,70 @@ class Functions extends Action
         }
     }
 
+    /**
+     * @param string $message
+     * @param Document $function
+     * @param string $trigger
+     * @param string $path
+     * @param string $method
+     * @param Document $user
+     * @param string|null $jwt
+     * @param string|null $event
+     * @throws Exception
+     */
+    private function fail(
+        string $message,
+        Database $dbForProject,
+        Document $function,
+        string $trigger,
+        string $path,
+        string $method,
+        Document $user,
+        string $jwt = null,
+        string $event = null,
+    ): void {
+        $headers['x-appwrite-trigger'] = $trigger;
+        $headers['x-appwrite-event'] = $event ?? '';
+        $headers['x-appwrite-user-id'] = $user->getId() ?? '';
+        $headers['x-appwrite-user-jwt'] = $jwt ?? '';
+
+        $headersFiltered = [];
+        foreach ($headers as $key => $value) {
+            if (\in_array(\strtolower($key), FUNCTION_ALLOWLIST_HEADERS_REQUEST)) {
+                $headersFiltered[] = ['name' => $key, 'value' => $value];
+            }
+        }
+
+        $executionId = ID::unique();
+        $execution = new Document([
+            '$id' => $executionId,
+            '$permissions' => $user->isEmpty() ? [] : [Permission::read(Role::user($user->getId()))],
+            'functionInternalId' => $function->getInternalId(),
+            'functionId' => $function->getId(),
+            'deploymentInternalId' => '',
+            'deploymentId' => '',
+            'trigger' => $trigger,
+            'status' => 'failed',
+            'responseStatusCode' => 0,
+            'responseHeaders' => [],
+            'requestPath' => $path,
+            'requestMethod' => $method,
+            'requestHeaders' => $headersFiltered,
+            'errors' => $message,
+            'logs' => '',
+            'duration' => 0.0,
+            'search' => implode(' ', [$function->getId(), $executionId]),
+        ]);
+
+        if ($function->getAttribute('logging')) {
+            $execution = $dbForProject->createDocument('executions', $execution);
+        }
+
+        if ($execution->isEmpty()) {
+            throw new Exception('Failed to create execution');
+        }
+    }
+
     /**
      * @param Log $log
      * @param Database $dbForProject
@@ -251,11 +315,15 @@ class Functions extends Action
         $deployment = $dbForProject->getDocument('deployments', $deploymentId);
 
         if ($deployment->getAttribute('resourceId') !== $functionId) {
-            throw new Exception('Deployment not found. Create deployment before trying to execute a function');
+            $errorMessage = 'The execution could not be completed because a corresponding deployment was not found. A function deployment needs to be created before it can be executed. Please create a deployment for your function and try again.';
+            $this->fail($errorMessage, $dbForProject, $function, $trigger, $path, $method, $user, $jwt, $event);
+            return;
         }
 
         if ($deployment->isEmpty()) {
-            throw new Exception('Deployment not found. Create deployment before trying to execute a function');
+            $errorMessage = 'The execution could not be completed because a corresponding deployment was not found. A function deployment needs to be created before it can be executed. Please create a deployment for your function and try again.';
+            $this->fail($errorMessage, $dbForProject, $function, $trigger, $path, $method, $user, $jwt, $event);
+            return;
         }
 
         $buildId = $deployment->getAttribute('buildId', '');
@@ -265,11 +333,15 @@ class Functions extends Action
         /** Check if build has exists */
         $build = $dbForProject->getDocument('builds', $buildId);
         if ($build->isEmpty()) {
-            throw new Exception('Build not found');
+            $errorMessage = 'The execution could not be completed because a corresponding deployment was not found. A function deployment needs to be created before it can be executed. Please create a deployment for your function and try again.';
+            $this->fail($errorMessage, $dbForProject, $function, $trigger, $path, $method, $user, $jwt, $event);
+            return;
         }
 
         if ($build->getAttribute('status') !== 'ready') {
-            throw new Exception('Build not ready');
+            $errorMessage = 'The execution could not be completed because the build is not ready. Please wait for the build to complete and try again.';
+            $this->fail($errorMessage, $dbForProject, $function, $trigger, $path, $method, $user, $jwt, $event);
+            return;
         }
 
         /** Check if  runtime is supported */
@@ -287,7 +359,6 @@ class Functions extends Action
         $headers['x-appwrite-user-id'] = $user->getId() ?? '';
         $headers['x-appwrite-user-jwt'] = $jwt ?? '';
 
-        /** Create execution or update execution status */
         /** Create execution or update execution status */
         $execution = $dbForProject->getDocument('executions', $executionId ?? '');
         if ($execution->isEmpty()) {

src/Appwrite/Platform/Workers/Messaging.php

@@ -25,7 +25,7 @@ use Utopia\Messaging\Adapter\SMS as SMSAdapter;
 use Utopia\Messaging\Adapter\SMS\Mock;
 use Utopia\Messaging\Adapter\SMS\Msg91;
 use Utopia\Messaging\Adapter\SMS\Telesign;
-use Utopia\Messaging\Adapter\SMS\Textmagic;
+use Utopia\Messaging\Adapter\SMS\TextMagic;
 use Utopia\Messaging\Adapter\SMS\Twilio;
 use Utopia\Messaging\Adapter\SMS\Vonage;
 use Utopia\Messaging\Messages\Email;
@@ -441,17 +441,24 @@ class Messaging extends Action
                 try {
                     $adapter->send($data);
 
+                    $countryCode = $adapter->getCountryCode($message['to'][0] ?? '');
+                    if (!empty($countryCode)) {
+                        $queueForUsage
+                            ->addMetric(str_replace('{countryCode}', $countryCode, METRIC_MESSAGES_COUNTRY_CODE), 1);
+                    }
                     $queueForUsage
                         ->addMetric(METRIC_MESSAGES, 1)
                         ->setProject($project)
                         ->trigger();
-                } catch (\Throwable $e) {
-                    throw new \Exception('Failed sending to targets with error: ' . $e->getMessage());
+                } catch (\Throwable $th) {
+                    throw new \Exception('Failed sending to targets with error: ' . $th->getMessage());
                 }
             };
         }, $batches));
     }
 
+
+
     private function getSmsAdapter(Document $provider): ?SMSAdapter
     {
         $credentials = $provider->getAttribute('credentials');
@@ -459,7 +466,7 @@ class Messaging extends Action
         return match ($provider->getAttribute('provider')) {
             'mock' => new Mock('username', 'password'),
             'twilio' => new Twilio($credentials['accountSid'], $credentials['authToken']),
-            'textmagic' => new Textmagic($credentials['username'], $credentials['apiKey']),
+            'textmagic' => new TextMagic($credentials['username'], $credentials['apiKey']),
             'telesign' => new Telesign($credentials['customerId'], $credentials['apiKey']),
             'msg91' => new Msg91($credentials['senderId'], $credentials['authKey'], $credentials['templateId']),
             'vonage' => new Vonage($credentials['apiKey'], $credentials['apiSecret']),

src/Appwrite/Platform/Workers/Migrations.php

@@ -16,6 +16,7 @@ use Utopia\Database\Exception\Restricted;
 use Utopia\Database\Exception\Structure;
 use Utopia\Database\Helpers\ID;
 use Utopia\Logger\Log;
+use Utopia\Logger\Log\Breadcrumb;
 use Utopia\Migration\Destinations\Appwrite as DestinationsAppwrite;
 use Utopia\Migration\Exception as MigrationException;
 use Utopia\Migration\Source;
@@ -85,6 +86,7 @@ class Migrations extends Action
             return;
         }
 
+        $log->addTag('migrationId', $migration->getId());
         $log->addTag('projectId', $project->getId());
 
         $this->processMigration($project, $migration, $log);
@@ -256,6 +258,7 @@ class Migrations extends Action
             $migrationDocument = $this->dbForProject->getDocument('migrations', $migration->getId());
             $migrationDocument->setAttribute('stage', 'processing');
             $migrationDocument->setAttribute('status', 'processing');
+            $log->addBreadcrumb(new Breadcrumb("debug", "migration", "Migration hit stage 'processing'", \microtime(true)));
             $this->updateMigrationDocument($migrationDocument, $projectDocument);
 
             $log->addTag('type', $migrationDocument->getAttribute('source'));
@@ -277,6 +280,7 @@ class Migrations extends Action
 
             /** Start Transfer */
             $migrationDocument->setAttribute('stage', 'migrating');
+            $log->addBreadcrumb(new Breadcrumb("debug", "migration", "Migration hit stage 'migrating'", \microtime(true)));
             $this->updateMigrationDocument($migrationDocument, $projectDocument);
             $transfer->run($migrationDocument->getAttribute('resources'), function () use ($migrationDocument, $transfer, $projectDocument) {
                 $migrationDocument->setAttribute('resourceData', json_encode($transfer->getCache()));
@@ -291,6 +295,7 @@ class Migrations extends Action
             if (!empty($sourceErrors) || !empty($destinationErrors)) {
                 $migrationDocument->setAttribute('status', 'failed');
                 $migrationDocument->setAttribute('stage', 'finished');
+                $log->addBreadcrumb(new Breadcrumb("debug", "migration", "Migration hit stage 'finished' and failed", \microtime(true)));
 
                 $errorMessages = [];
                 foreach ($sourceErrors as $error) {
@@ -303,6 +308,7 @@ class Migrations extends Action
                 }
 
                 $migrationDocument->setAttribute('errors', $errorMessages);
+                $log->addExtra('migrationErrors', json_encode($errorMessages));
                 $this->updateMigrationDocument($migrationDocument, $projectDocument);
 
                 return;
@@ -310,6 +316,7 @@ class Migrations extends Action
 
             $migrationDocument->setAttribute('status', 'completed');
             $migrationDocument->setAttribute('stage', 'finished');
+            $log->addBreadcrumb(new Breadcrumb("debug", "migration", "Migration hit stage 'finished' and succeeded", \microtime(true)));
         } catch (\Throwable $th) {
             Console::error($th->getMessage());
 
@@ -338,6 +345,7 @@ class Migrations extends Action
                 }
 
                 $migrationDocument->setAttribute('errors', $errorMessages);
+                $log->addTag('migrationErrors', json_encode($errorMessages));
             }
         } finally {
             if ($tempAPIKey) {
@@ -347,7 +355,7 @@ class Migrations extends Action
                 $this->updateMigrationDocument($migrationDocument, $projectDocument);
 
                 if ($migrationDocument->getAttribute('status', '') == 'failed') {
-                    throw new Exception(implode("\n", $migrationDocument->getAttribute('errors', [])));
+                    throw new Exception("Migration failed");
                 }
             }
         }

src/Appwrite/Specification/Format/OpenAPI3.php

@@ -96,15 +96,15 @@ class OpenAPI3 extends Format
         ];
 
         if (isset($output['components']['securitySchemes']['Project'])) {
-            $output['components']['securitySchemes']['Project']['x-appwrite'] = ['demo' => '5df5acd0d48c2'];
+            $output['components']['securitySchemes']['Project']['x-appwrite'] = ['demo' => '<YOUR_PROJECT_ID>'];
         }
 
         if (isset($output['components']['securitySchemes']['Key'])) {
-            $output['components']['securitySchemes']['Key']['x-appwrite'] = ['demo' => '919c2d18fb5d4...a2ae413da83346ad2'];
+            $output['components']['securitySchemes']['Key']['x-appwrite'] = ['demo' => '<YOUR_API_KEY>'];
         }
 
         if (isset($output['securityDefinitions']['JWT'])) {
-            $output['securityDefinitions']['JWT']['x-appwrite'] = ['demo' => 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJ...'];
+            $output['securityDefinitions']['JWT']['x-appwrite'] = ['demo' => '<YOUR_JWT>'];
         }
 
         if (isset($output['components']['securitySchemes']['Locale'])) {

src/Appwrite/Specification/Format/Swagger2.php

@@ -93,15 +93,15 @@ class Swagger2 extends Format
         ];
 
         if (isset($output['securityDefinitions']['Project'])) {
-            $output['securityDefinitions']['Project']['x-appwrite'] = ['demo' => '5df5acd0d48c2'];
+            $output['securityDefinitions']['Project']['x-appwrite'] = ['demo' => '<YOUR_PROJECT_ID>'];
         }
 
         if (isset($output['securityDefinitions']['Key'])) {
-            $output['securityDefinitions']['Key']['x-appwrite'] = ['demo' => '919c2d18fb5d4...a2ae413da83346ad2'];
+            $output['securityDefinitions']['Key']['x-appwrite'] = ['demo' => '<YOUR_API_KEY>'];
         }
 
         if (isset($output['securityDefinitions']['JWT'])) {
-            $output['securityDefinitions']['JWT']['x-appwrite'] = ['demo' => 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJ...'];
+            $output['securityDefinitions']['JWT']['x-appwrite'] = ['demo' => '<YOUR_JWT>'];
         }
 
         if (isset($output['securityDefinitions']['Locale'])) {

src/Appwrite/Vcs/Comment.php

@@ -67,9 +67,6 @@ class Comment
             ];
         }
 
-        //TODO: Update link to documentation
-        $text .= "**Your function has been automatically deployed.** Learn more about Appwrite [Function Deployments](https://appwrite.io/docs/functions).\n\n";
-
         foreach ($projects as $projectId => $project) {
             $text .= "**{$project['name']}** `{$projectId}`\n\n";
             $text .= "| Function | ID | Status | Action |\n";
@@ -79,6 +76,18 @@ class Comment
             $hostname = System::getEnv('_APP_DOMAIN');
 
             foreach ($project['functions'] as $functionId => $function) {
+                if ($function['status'] === 'waiting' || $function['status'] === 'processing' || $function['status'] === 'building') {
+                    $text .= "**Your function deployment is in progress. Please check back in a few minutes for the updated status.**\n\n";
+                } elseif ($function['status'] === 'ready') {
+                    $text .= "**Your function has been successfully deployed.**\n\n";
+                } else {
+                    $text .= "**Your function deployment has failed. Please check the logs for more details and retry.**\n\n";
+                }
+
+                $text .= "Project name: **{$project['name']}** \nProject ID: `{$projectId}`\n\n";
+                $text .= "| Function | ID | Status | Action |\n";
+                $text .= "| :- | :-  | :-  | :- |\n";
+
                 $generateImage = function (string $status) use ($protocol, $hostname) {
                     $extention = $status === 'building' ? 'gif' : 'png';
                     $imagesUrl = $protocol . '://' . $hostname . '/images/vcs/';
@@ -106,7 +115,9 @@ class Comment
 
             $text .= "\n\n";
         }
-        //TODO: Update did you know section
+        $functionUrl = $protocol . '://' . $hostname . '/console/project-' . $projectId . '/functions/function-' . $functionId;
+        $text .= "Only deployments on the production branch are activated automatically. If you'd like to activate this deployment, navigate to [your deployments]($functionUrl). Learn more about Appwrite [Function deployments](https://appwrite.io/docs/functions).\n\n";
+
         $tip = $this->tips[array_rand($this->tips)];
         $text .= "> **💡 Did you know?** \n " . $tip . "\n\n";
 

tests/e2e/General/HTTPTest.php

@@ -31,7 +31,7 @@ class HTTPTest extends Scope
         $this->assertEquals(204, $response['headers']['status-code']);
         $this->assertEquals('Appwrite', $response['headers']['server']);
         $this->assertEquals('GET, POST, PUT, PATCH, DELETE', $response['headers']['access-control-allow-methods']);
-        $this->assertEquals('Origin, Cookie, Set-Cookie, X-Requested-With, Content-Type, Access-Control-Allow-Origin, Access-Control-Request-Headers, Accept, X-Appwrite-Project, X-Appwrite-Key, X-Appwrite-Locale, X-Appwrite-Mode, X-Appwrite-JWT, X-Appwrite-Response-Format, X-Appwrite-Timeout, X-SDK-Version, X-SDK-Name, X-SDK-Language, X-SDK-Platform, X-SDK-GraphQL, X-Appwrite-ID, X-Appwrite-Timestamp, Content-Range, Range, Cache-Control, Expires, Pragma, X-Appwrite-Session, X-Fallback-Cookies, X-Forwarded-For, X-Forwarded-User-Agent', $response['headers']['access-control-allow-headers']);
+        $this->assertEquals('Origin, Cookie, Set-Cookie, X-Requested-With, Content-Type, Access-Control-Allow-Origin, Access-Control-Request-Headers, Accept, X-Appwrite-Project, X-Appwrite-Key, X-Appwrite-Locale, X-Appwrite-Mode, X-Appwrite-JWT, X-Appwrite-Response-Format, X-Appwrite-Timeout, X-Appwrite-Shared-Tables, X-SDK-Version, X-SDK-Name, X-SDK-Language, X-SDK-Platform, X-SDK-GraphQL, X-Appwrite-ID, X-Appwrite-Timestamp, Content-Range, Range, Cache-Control, Expires, Pragma, X-Appwrite-Session, X-Fallback-Cookies, X-Forwarded-For, X-Forwarded-User-Agent', $response['headers']['access-control-allow-headers']);
         $this->assertEquals('X-Appwrite-Session, X-Fallback-Cookies', $response['headers']['access-control-expose-headers']);
         $this->assertEquals('http://localhost', $response['headers']['access-control-allow-origin']);
         $this->assertEquals('true', $response['headers']['access-control-allow-credentials']);

tests/e2e/Services/Console/ModeTest.php

@@ -0,0 +1,34 @@
+<?php
+
+namespace Tests\E2E\Services\Console;
+
+use Appwrite\Extend\Exception;
+use Tests\E2E\Client;
+use Tests\E2E\Scopes\ProjectConsole;
+use Tests\E2E\Scopes\Scope;
+use Tests\E2E\Scopes\SideClient;
+
+class ModeTest extends Scope
+{
+    use ProjectConsole;
+    use SideClient;
+
+    public function testConsoleWithAdminMode(): void
+    {
+        $this->markTestSkipped();
+
+        /**
+         * Test for SUCCESS
+         */
+        /*
+         $response = $this->client->call(Client::METHOD_GET, '/account', array_merge([
+             'content-type' => 'application/json',
+             'x-appwrite-project' => $this->getProject()['$id'],
+             'x-appwrite-mode' => 'admin',
+         ], $this->getHeaders()));
+
+         $this->assertEquals(400, $response['headers']['status-code']);
+         $this->assertEquals(Exception::GENERAL_BAD_REQUEST, $response['body']['type']);
+         */
+    }
+}

tests/e2e/Services/Databases/DatabasesBase.php

@@ -85,6 +85,36 @@ trait DatabasesBase
         ];
     }
 
+    /**
+     * @depends testCreateCollection
+     */
+    public function testConsoleProject(array $data)
+    {
+        $response = $this->client->call(
+            Client::METHOD_GET,
+            '/databases/console/collections/' . $data['moviesId'] . '/documents',
+            array_merge([
+                'content-type' => 'application/json',
+                'x-appwrite-project' => 'console',
+            ], $this->getHeaders())
+        );
+
+        $this->assertEquals(401, $response['headers']['status-code']);
+        $this->assertEquals('general_access_forbidden', $response['body']['type']);
+        $this->assertEquals('This endpoint is not available for the console project. The Appwrite Console is a reserved project ID and cannot be used with the Appwrite SDKs and APIs. Please check if your project ID is correct.', $response['body']['message']);
+
+        $response = $this->client->call(
+            Client::METHOD_GET,
+            '/databases/console/collections/' . $data['moviesId'] . '/documents',
+            array_merge([
+                'content-type' => 'application/json',
+                // 'x-appwrite-project' => '', empty header
+            ], $this->getHeaders())
+        );
+        $this->assertEquals(401, $response['headers']['status-code']);
+        $this->assertEquals('No Appwrite project was specified. Please specify your project ID when initializing your Appwrite SDK.', $response['body']['message']);
+    }
+
     /**
      * @depends testCreateCollection
      */
@@ -2098,16 +2128,17 @@ trait DatabasesBase
         // Todo: Not sure what to do we with Query length Test VS old? JSON validator will fails if query string will be truncated?
         //$this->assertEquals(400, $documents['headers']['status-code']);
 
-        $documents = $this->client->call(Client::METHOD_GET, '/databases/' . $databaseId . '/collections/' . $data['moviesId'] . '/documents', array_merge([
-            'content-type' => 'application/json',
-            'x-appwrite-project' => $this->getProject()['$id'],
-        ], $this->getHeaders()), [
-            'queries' => [
-                Query::search('actors', 'Tom')->toString(),
-            ],
-        ]);
-        $this->assertEquals(400, $documents['headers']['status-code']);
-        $this->assertEquals('Invalid query: Cannot query search on attribute "actors" because it is an array.', $documents['body']['message']);
+        // Todo: Disabled for CL - Uncomment after ProxyDatabase cleanup for find method
+        // $documents = $this->client->call(Client::METHOD_GET, '/databases/' . $databaseId . '/collections/' . $data['moviesId'] . '/documents', array_merge([
+        //     'content-type' => 'application/json',
+        //     'x-appwrite-project' => $this->getProject()['$id'],
+        // ], $this->getHeaders()), [
+        //     'queries' => [
+        //         Query::search('actors', 'Tom')->toString(),
+        //     ],
+        // ]);
+        // $this->assertEquals(400, $documents['headers']['status-code']);
+        // $this->assertEquals('Invalid query: Cannot query search on attribute "actors" because it is an array.', $documents['body']['message']);
 
         return [];
     }
@@ -4371,7 +4402,7 @@ trait DatabasesBase
                 Query::isNotNull('$id')->toString(),
                 Query::startsWith('fullName', 'Stevie')->toString(),
                 Query::endsWith('fullName', 'Wonder')->toString(),
-                Query::between('$createdAt', '1975-12-06', '2050-12-0')->toString(),
+                Query::between('$createdAt', '1975-12-06', '2050-12-01')->toString(),
             ],
         ]);
 
@@ -4730,7 +4761,7 @@ trait DatabasesBase
             ], $this->getHeaders()), [
                 'documentId' => ID::unique(),
                 'data' => [
-                    'longtext' => file_get_contents('tests/resources/longtext.txt'),
+                    'longtext' => file_get_contents(__DIR__ . '/../../../resources/longtext.txt'),
                 ],
                 'permissions' => [
                     Permission::read(Role::user($this->getUser()['$id'])),

tests/e2e/Services/Databases/DatabasesCustomServerTest.php

@@ -1296,8 +1296,6 @@ class DatabasesCustomServerTest extends Scope
             'x-appwrite-key' => $this->getProject()['apiKey'],
         ], $this->getHeaders()));
 
-        \var_dump($attributes['body']);
-
         $this->assertEquals(0, $attributes['body']['total']);
     }
 
@@ -1438,10 +1436,9 @@ class DatabasesCustomServerTest extends Scope
         }
 
         // Test indexLimit = 64
-        // MariaDB, MySQL, and MongoDB create 5 indexes per new collection
+        // MariaDB, MySQL, and MongoDB create 6 indexes per new collection
         // Add up to the limit, then check if the next index throws IndexLimitException
         for ($i = 0; $i < 58; $i++) {
-            // $this->assertEquals(true, static::getDatabase()->createIndex('indexLimit', "index{$i}", Database::INDEX_KEY, ["test{$i}"], [16]));
             $index = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections/' . $collectionId . '/indexes', array_merge([
                 'content-type' => 'application/json',
                 'x-appwrite-project' => $this->getProject()['$id'],

tests/e2e/Services/GraphQL/TeamsServerTest.php

@@ -297,6 +297,7 @@ class TeamsServerTest extends Scope
         $this->assertEquals(204, $team['headers']['status-code']);
     }
 
+    /** @group cl-ignore */
     public function testDeleteTeam()
     {
         $team = $this->testCreateTeam();

tests/e2e/Services/Locale/LocaleBase.php

@@ -227,9 +227,9 @@ trait LocaleBase
         /**
          * Test for SUCCESS
          */
-        $languages           = require('app/config/locale/codes.php');
-        $defaultCountries    = require('app/config/locale/countries.php');
-        $defaultContinents   = require('app/config/locale/continents.php');
+        $languages           = require(__DIR__ . '/../../../../app/config/locale/codes.php');
+        $defaultCountries    = require(__DIR__ . '/../../../../app/config/locale/countries.php');
+        $defaultContinents   = require(__DIR__ . '/../../../../app/config/locale/continents.php');
 
         foreach ($languages as $lang) {
             $response = $this->client->call(Client::METHOD_GET, '/locale/countries', [

tests/e2e/Services/Projects/ProjectsConsoleClientTest.php

@@ -9,6 +9,7 @@ use Tests\E2E\General\UsageTest;
 use Tests\E2E\Scopes\ProjectConsole;
 use Tests\E2E\Scopes\Scope;
 use Tests\E2E\Scopes\SideClient;
+use Utopia\Database\Database;
 use Utopia\Database\DateTime;
 use Utopia\Database\Document;
 use Utopia\Database\Helpers\ID;
@@ -950,7 +951,7 @@ class ProjectsConsoleClientTest extends Scope
     public function testUpdateProjectOAuth($data): array
     {
         $id = $data['projectId'] ?? '';
-        $providers = require('app/config/oAuthProviders.php');
+        $providers = require(__DIR__ . '/../../../../app/config/oAuthProviders.php');
 
         /**
          * Test for SUCCESS
@@ -1061,7 +1062,7 @@ class ProjectsConsoleClientTest extends Scope
     public function testUpdateProjectAuthStatus($data): array
     {
         $id = $data['projectId'] ?? '';
-        $auth = require('app/config/auth.php');
+        $auth = require(__DIR__ . '/../../../../app/config/auth.php');
 
         $originalEmail = uniqid() . 'user@localhost.test';
         $originalPassword = 'password';
@@ -1281,7 +1282,9 @@ class ProjectsConsoleClientTest extends Scope
             'name' => $name,
         ]);
 
-        $this->assertEquals($response['headers']['status-code'], 501);
+        $this->assertEquals(Exception::USER_COUNT_EXCEEDED, $response['body']['type']);
+        $this->assertEquals(400, $response['headers']['status-code']);
+
 
         /**
          * Test for FAILURE
@@ -1927,7 +1930,7 @@ class ProjectsConsoleClientTest extends Scope
         $this->assertNotEmpty($project['body']['$id']);
 
         $id = $project['body']['$id'];
-        $services = require('app/config/services.php');
+        $services = require(__DIR__ . '/../../../../app/config/services.php');
 
         /**
          * Test for Disabled
@@ -2001,7 +2004,7 @@ class ProjectsConsoleClientTest extends Scope
     {
         $id = $data['projectId'];
 
-        $services = require('app/config/services.php');
+        $services = require(__DIR__ . '/../../../../app/config/services.php');
 
         /**
          * Test for Disabled
@@ -2075,7 +2078,7 @@ class ProjectsConsoleClientTest extends Scope
     {
         $id = $data['projectId'];
 
-        $services = require('app/config/services.php');
+        $services = require(__DIR__ . '/../../../../app/config/services.php');
 
         /**
          * Test for Disabled
@@ -3491,4 +3494,504 @@ class ProjectsConsoleClientTest extends Scope
 
         return $data;
     }
+
+    public function testTenantIsolation(): void
+    {
+        // Create a team and a project
+        $team = $this->client->call(Client::METHOD_POST, '/teams', array_merge([
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $this->getProject()['$id'],
+        ], $this->getHeaders()), [
+            'teamId' => ID::unique(),
+            'name' => 'Amazing Team',
+        ]);
+
+        $teamId = $team['body']['$id'];
+
+        // Project-level isolation
+        $project1 = $this->client->call(Client::METHOD_POST, '/projects', array_merge([
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $this->getProject()['$id'],
+            'x-appwrite-shared-tables' => false
+        ], $this->getHeaders()), [
+            'projectId' => ID::unique(),
+            'name' => 'Amazing Project',
+            'teamId' => $teamId,
+            'region' => 'default'
+        ]);
+
+        // Application level isolation (shared tables)
+        $project2 = $this->client->call(Client::METHOD_POST, '/projects', array_merge([
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $this->getProject()['$id'],
+            'x-appwrite-shared-tables' => true
+        ], $this->getHeaders()), [
+            'projectId' => ID::unique(),
+            'name' => 'Amazing Project',
+            'teamId' => $teamId,
+            'region' => 'default'
+        ]);
+
+        // Project-level isolation
+        $project3 = $this->client->call(Client::METHOD_POST, '/projects', array_merge([
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $this->getProject()['$id'],
+            'x-appwrite-shared-tables' => false
+        ], $this->getHeaders()), [
+            'projectId' => ID::unique(),
+            'name' => 'Amazing Project',
+            'teamId' => $teamId,
+            'region' => 'default'
+        ]);
+
+        // Application level isolation (shared tables)
+        $project4 = $this->client->call(Client::METHOD_POST, '/projects', array_merge([
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $this->getProject()['$id'],
+            'x-appwrite-shared-tables' => true
+        ], $this->getHeaders()), [
+            'projectId' => ID::unique(),
+            'name' => 'Amazing Project',
+            'teamId' => $teamId,
+            'region' => 'default'
+        ]);
+
+        // Create and API key in each project
+        $key1 = $this->client->call(Client::METHOD_POST, '/projects/' . $project1['body']['$id'] . '/keys', array_merge([
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $this->getProject()['$id'],
+        ], $this->getHeaders()), [
+            'name' => 'Key Test',
+            'scopes' => ['databases.read', 'databases.write', 'collections.read', 'collections.write', 'attributes.read', 'attributes.write', 'indexes.read', 'indexes.write', 'documents.read', 'documents.write', 'users.read', 'users.write'],
+        ]);
+
+        $key2 = $this->client->call(Client::METHOD_POST, '/projects/' . $project2['body']['$id'] . '/keys', array_merge([
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $this->getProject()['$id'],
+        ], $this->getHeaders()), [
+            'name' => 'Key Test',
+            'scopes' => ['databases.read', 'databases.write', 'collections.read', 'collections.write', 'attributes.read', 'attributes.write', 'indexes.read', 'indexes.write', 'documents.read', 'documents.write', 'users.read', 'users.write'],
+        ]);
+
+        $key3 = $this->client->call(Client::METHOD_POST, '/projects/' . $project3['body']['$id'] . '/keys', array_merge([
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $this->getProject()['$id'],
+        ], $this->getHeaders()), [
+            'name' => 'Key Test',
+            'scopes' => ['databases.read', 'databases.write', 'collections.read', 'collections.write', 'attributes.read', 'attributes.write', 'indexes.read', 'indexes.write', 'documents.read', 'documents.write', 'users.read', 'users.write'],
+        ]);
+
+        $key4 = $this->client->call(Client::METHOD_POST, '/projects/' . $project4['body']['$id'] . '/keys', array_merge([
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $this->getProject()['$id'],
+        ], $this->getHeaders()), [
+            'name' => 'Key Test',
+            'scopes' => ['databases.read', 'databases.write', 'collections.read', 'collections.write', 'attributes.read', 'attributes.write', 'indexes.read', 'indexes.write', 'documents.read', 'documents.write', 'users.read', 'users.write'],
+        ]);
+
+        // Create a database in each project
+        $database1 = $this->client->call(Client::METHOD_POST, '/databases', [
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $project1['body']['$id'],
+            'x-appwrite-key' => $key1['body']['secret']
+        ], [
+            'databaseId' => ID::unique(),
+            'name' => 'Amazing Database',
+        ]);
+
+        $database2 = $this->client->call(Client::METHOD_POST, '/databases', [
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $project2['body']['$id'],
+            'x-appwrite-key' => $key2['body']['secret']
+        ], [
+            'databaseId' => ID::unique(),
+            'name' => 'Amazing Database',
+        ]);
+
+        $database3 = $this->client->call(Client::METHOD_POST, '/databases', [
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $project3['body']['$id'],
+            'x-appwrite-key' => $key3['body']['secret']
+        ], [
+            'databaseId' => ID::unique(),
+            'name' => 'Amazing Database',
+        ]);
+
+        $database4 = $this->client->call(Client::METHOD_POST, '/databases', [
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $project4['body']['$id'],
+            'x-appwrite-key' => $key4['body']['secret']
+        ], [
+            'databaseId' => ID::unique(),
+            'name' => 'Amazing Database',
+        ]);
+
+        // Create a collection in each project
+        $collection1 = $this->client->call(Client::METHOD_POST, '/databases/' . $database1['body']['$id'] . '/collections', [
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $project1['body']['$id'],
+            'x-appwrite-key' => $key1['body']['secret']
+        ], [
+            'databaseId' => $database1['body']['$id'],
+            'collectionId' => ID::unique(),
+            'name' => 'Amazing Collection',
+        ]);
+
+        $collection2 = $this->client->call(Client::METHOD_POST, '/databases/' . $database2['body']['$id'] . '/collections', [
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $project2['body']['$id'],
+            'x-appwrite-key' => $key2['body']['secret']
+        ], [
+            'databaseId' => $database2['body']['$id'],
+            'collectionId' => ID::unique(),
+            'name' => 'Amazing Collection',
+        ]);
+
+        $collection3 = $this->client->call(Client::METHOD_POST, '/databases/' . $database3['body']['$id'] . '/collections', [
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $project3['body']['$id'],
+            'x-appwrite-key' => $key3['body']['secret']
+        ], [
+            'databaseId' => $database3['body']['$id'],
+            'collectionId' => ID::unique(),
+            'name' => 'Amazing Collection',
+        ]);
+
+        $collection4 = $this->client->call(Client::METHOD_POST, '/databases/' . $database4['body']['$id'] . '/collections', [
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $project4['body']['$id'],
+            'x-appwrite-key' => $key4['body']['secret']
+        ], [
+            'databaseId' => $database4['body']['$id'],
+            'collectionId' => ID::unique(),
+            'name' => 'Amazing Collection',
+        ]);
+
+        // Create an attribute in each project
+        $attribute1 = $this->client->call(Client::METHOD_POST, '/databases/' . $database1['body']['$id'] . '/collections/' . $collection1['body']['$id'] . '/attributes/string', [
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $project1['body']['$id'],
+            'x-appwrite-key' => $key1['body']['secret']
+        ], [
+            'databaseId' => $database1['body']['$id'],
+            'collectionId' => $collection1['body']['$id'],
+            'key' => ID::unique(),
+            'size' => 255,
+            'required' => true
+        ]);
+
+        $attribute2 = $this->client->call(Client::METHOD_POST, '/databases/' . $database2['body']['$id'] . '/collections/' . $collection2['body']['$id'] . '/attributes/string', [
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $project2['body']['$id'],
+            'x-appwrite-key' => $key2['body']['secret']
+        ], [
+            'databaseId' => $database2['body']['$id'],
+            'collectionId' => $collection2['body']['$id'],
+            'key' => ID::unique(),
+            'size' => 255,
+            'required' => true
+        ]);
+
+        $attribute3 = $this->client->call(Client::METHOD_POST, '/databases/' . $database3['body']['$id'] . '/collections/' . $collection3['body']['$id'] . '/attributes/string', [
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $project3['body']['$id'],
+            'x-appwrite-key' => $key3['body']['secret']
+        ], [
+            'databaseId' => $database3['body']['$id'],
+            'collectionId' => $collection3['body']['$id'],
+            'key' => ID::unique(),
+            'size' => 255,
+            'required' => true
+        ]);
+
+        $attribute4 = $this->client->call(Client::METHOD_POST, '/databases/' . $database4['body']['$id'] . '/collections/' . $collection4['body']['$id'] . '/attributes/string', [
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $project4['body']['$id'],
+            'x-appwrite-key' => $key4['body']['secret']
+        ], [
+            'databaseId' => $database4['body']['$id'],
+            'collectionId' => $collection4['body']['$id'],
+            'key' => ID::unique(),
+            'size' => 255,
+            'required' => true
+        ]);
+
+        // Wait for attributes
+        \sleep(2);
+
+        // Create an index in each project
+        $index1 = $this->client->call(Client::METHOD_POST, '/databases/' . $database1['body']['$id'] . '/collections/' . $collection1['body']['$id'] . '/indexes', [
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $project1['body']['$id'],
+            'x-appwrite-key' => $key1['body']['secret']
+        ], [
+            'databaseId' => $database1['body']['$id'],
+            'collectionId' => $collection1['body']['$id'],
+            'key' => ID::unique(),
+            'type' => Database::INDEX_KEY,
+            'attributes' => [$attribute1['body']['key']],
+        ]);
+
+        $index2 = $this->client->call(Client::METHOD_POST, '/databases/' . $database2['body']['$id'] . '/collections/' . $collection2['body']['$id'] . '/indexes', [
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $project2['body']['$id'],
+            'x-appwrite-key' => $key2['body']['secret']
+        ], [
+            'databaseId' => $database2['body']['$id'],
+            'collectionId' => $collection2['body']['$id'],
+            'key' => ID::unique(),
+            'type' => Database::INDEX_KEY,
+            'attributes' => [$attribute2['body']['key']],
+        ]);
+
+        $index3 = $this->client->call(Client::METHOD_POST, '/databases/' . $database3['body']['$id'] . '/collections/' . $collection3['body']['$id'] . '/indexes', [
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $project3['body']['$id'],
+            'x-appwrite-key' => $key3['body']['secret']
+        ], [
+            'databaseId' => $database3['body']['$id'],
+            'collectionId' => $collection3['body']['$id'],
+            'key' => ID::unique(),
+            'type' => Database::INDEX_KEY,
+            'attributes' => [$attribute3['body']['key']],
+        ]);
+
+        $index4 = $this->client->call(Client::METHOD_POST, '/databases/' . $database4['body']['$id'] . '/collections/' . $collection4['body']['$id'] . '/indexes', [
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $project4['body']['$id'],
+            'x-appwrite-key' => $key4['body']['secret']
+        ], [
+            'databaseId' => $database4['body']['$id'],
+            'collectionId' => $collection4['body']['$id'],
+            'key' => ID::unique(),
+            'type' => Database::INDEX_KEY,
+            'attributes' => [$attribute4['body']['key']],
+        ]);
+
+        // Wait for indexes
+        \sleep(2);
+
+        // Assert that each project has only 1 database, 1 collection, 1 attribute and 1 index
+        $databasesProject1 = $this->client->call(Client::METHOD_GET, '/databases', [
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $project1['body']['$id'],
+            'x-appwrite-key' => $key1['body']['secret']
+        ]);
+
+        $this->assertEquals(1, $databasesProject1['body']['total']);
+        $this->assertEquals(1, \count($databasesProject1['body']['databases']));
+
+        $databasesProject2 = $this->client->call(Client::METHOD_GET, '/databases', [
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $project2['body']['$id'],
+            'x-appwrite-key' => $key2['body']['secret']
+        ]);
+
+        $this->assertEquals(1, $databasesProject2['body']['total']);
+        $this->assertEquals(1, \count($databasesProject2['body']['databases']));
+
+        $databasesProject3 = $this->client->call(Client::METHOD_GET, '/databases', [
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $project3['body']['$id'],
+            'x-appwrite-key' => $key3['body']['secret']
+        ]);
+
+        $this->assertEquals(1, $databasesProject3['body']['total']);
+        $this->assertEquals(1, \count($databasesProject3['body']['databases']));
+
+        $databasesProject4 = $this->client->call(Client::METHOD_GET, '/databases', [
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $project4['body']['$id'],
+            'x-appwrite-key' => $key4['body']['secret']
+        ]);
+
+        $this->assertEquals(1, $databasesProject4['body']['total']);
+        $this->assertEquals(1, \count($databasesProject4['body']['databases']));
+
+        $collectionsProject1 = $this->client->call(Client::METHOD_GET, '/databases/' . $database1['body']['$id'] . '/collections', [
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $project1['body']['$id'],
+            'x-appwrite-key' => $key1['body']['secret']
+        ]);
+
+        $this->assertEquals(1, $collectionsProject1['body']['total']);
+        $this->assertEquals(1, \count($collectionsProject1['body']['collections']));
+
+        $collectionsProject2 = $this->client->call(Client::METHOD_GET, '/databases/' . $database2['body']['$id'] . '/collections', [
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $project2['body']['$id'],
+            'x-appwrite-key' => $key2['body']['secret']
+        ]);
+
+        $this->assertEquals(1, $collectionsProject2['body']['total']);
+        $this->assertEquals(1, \count($collectionsProject2['body']['collections']));
+
+        $collectionsProject3 = $this->client->call(Client::METHOD_GET, '/databases/' . $database3['body']['$id'] . '/collections', [
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $project3['body']['$id'],
+            'x-appwrite-key' => $key3['body']['secret']
+        ]);
+
+        $this->assertEquals(1, $collectionsProject3['body']['total']);
+        $this->assertEquals(1, \count($collectionsProject3['body']['collections']));
+
+        $collectionsProject4 = $this->client->call(Client::METHOD_GET, '/databases/' . $database4['body']['$id'] . '/collections', [
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $project4['body']['$id'],
+            'x-appwrite-key' => $key4['body']['secret']
+        ]);
+
+        $this->assertEquals(1, $collectionsProject4['body']['total']);
+        $this->assertEquals(1, \count($collectionsProject4['body']['collections']));
+
+        $attributesProject1 = $this->client->call(Client::METHOD_GET, '/databases/' . $database1['body']['$id'] . '/collections/' . $collection1['body']['$id'] . '/attributes', [
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $project1['body']['$id'],
+            'x-appwrite-key' => $key1['body']['secret']
+        ]);
+
+        $this->assertEquals(1, $attributesProject1['body']['total']);
+        $this->assertEquals(1, \count($attributesProject1['body']['attributes']));
+        $this->assertEquals('available', $attributesProject1['body']['attributes'][0]['status']);
+
+        $attributesProject2 = $this->client->call(Client::METHOD_GET, '/databases/' . $database2['body']['$id'] . '/collections/' . $collection2['body']['$id'] . '/attributes', [
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $project2['body']['$id'],
+            'x-appwrite-key' => $key2['body']['secret']
+        ]);
+
+        $this->assertEquals(1, $attributesProject2['body']['total']);
+        $this->assertEquals(1, \count($attributesProject2['body']['attributes']));
+        $this->assertEquals('available', $attributesProject2['body']['attributes'][0]['status']);
+
+        $attributesProject3 = $this->client->call(Client::METHOD_GET, '/databases/' . $database3['body']['$id'] . '/collections/' . $collection3['body']['$id'] . '/attributes', [
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $project3['body']['$id'],
+            'x-appwrite-key' => $key3['body']['secret']
+        ]);
+
+        $this->assertEquals(1, $attributesProject3['body']['total']);
+        $this->assertEquals(1, \count($attributesProject3['body']['attributes']));
+        $this->assertEquals('available', $attributesProject3['body']['attributes'][0]['status']);
+
+        $attributesProject4 = $this->client->call(Client::METHOD_GET, '/databases/' . $database4['body']['$id'] . '/collections/' . $collection4['body']['$id'] . '/attributes', [
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $project4['body']['$id'],
+            'x-appwrite-key' => $key4['body']['secret']
+        ]);
+
+        $this->assertEquals(1, $attributesProject4['body']['total']);
+        $this->assertEquals(1, \count($attributesProject4['body']['attributes']));
+        $this->assertEquals('available', $attributesProject4['body']['attributes'][0]['status']);
+
+        $indexesProject1 = $this->client->call(Client::METHOD_GET, '/databases/' . $database1['body']['$id'] . '/collections/' . $collection1['body']['$id'] . '/indexes', [
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $project1['body']['$id'],
+            'x-appwrite-key' => $key1['body']['secret']
+        ]);
+
+        $this->assertEquals(1, $indexesProject1['body']['total']);
+        $this->assertEquals(1, \count($indexesProject1['body']['indexes']));
+
+        $indexesProject2 = $this->client->call(Client::METHOD_GET, '/databases/' . $database2['body']['$id'] . '/collections/' . $collection2['body']['$id'] . '/indexes', [
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $project2['body']['$id'],
+            'x-appwrite-key' => $key2['body']['secret']
+        ]);
+
+        $this->assertEquals(1, $indexesProject2['body']['total']);
+        $this->assertEquals(1, \count($indexesProject2['body']['indexes']));
+
+        $indexesProject3 = $this->client->call(Client::METHOD_GET, '/databases/' . $database3['body']['$id'] . '/collections/' . $collection3['body']['$id'] . '/indexes', [
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $project3['body']['$id'],
+            'x-appwrite-key' => $key3['body']['secret']
+        ]);
+
+        $this->assertEquals(1, $indexesProject3['body']['total']);
+        $this->assertEquals(1, \count($indexesProject3['body']['indexes']));
+
+        $indexesProject4 = $this->client->call(Client::METHOD_GET, '/databases/' . $database4['body']['$id'] . '/collections/' . $collection4['body']['$id'] . '/indexes', [
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $project4['body']['$id'],
+            'x-appwrite-key' => $key4['body']['secret']
+        ]);
+
+        $this->assertEquals(1, $indexesProject4['body']['total']);
+        $this->assertEquals(1, \count($indexesProject4['body']['indexes']));
+
+        // Attempt to read cross-type resources
+        $collectionProject2WithProject1Key = $this->client->call(Client::METHOD_GET, '/databases/' . $database2['body']['$id'] . '/collections/' . $collection2['body']['$id'], [
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $project1['body']['$id'],
+            'x-appwrite-key' => $key1['body']['secret']
+        ]);
+
+        $this->assertEquals(404, $collectionProject2WithProject1Key['headers']['status-code']);
+
+        $collectionProject1WithProject2Key = $this->client->call(Client::METHOD_GET, '/databases/' . $database1['body']['$id'] . '/collections/' . $collection1['body']['$id'], [
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $project2['body']['$id'],
+            'x-appwrite-key' => $key2['body']['secret']
+        ]);
+
+        $this->assertEquals(404, $collectionProject1WithProject2Key['headers']['status-code']);
+
+        // Attempt to read cross-tenant resources
+        $collectionProject3WithProject1Key = $this->client->call(Client::METHOD_GET, '/databases/' . $database3['body']['$id'] . '/collections/' . $collection3['body']['$id'], [
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $project1['body']['$id'],
+            'x-appwrite-key' => $key1['body']['secret']
+        ]);
+
+        $this->assertEquals(404, $collectionProject3WithProject1Key['headers']['status-code']);
+
+        $collectionProject1WithProject3Key = $this->client->call(Client::METHOD_GET, '/databases/' . $database1['body']['$id'] . '/collections/' . $collection1['body']['$id'], [
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $project3['body']['$id'],
+            'x-appwrite-key' => $key3['body']['secret']
+        ]);
+
+        $this->assertEquals(404, $collectionProject1WithProject3Key['headers']['status-code']);
+
+        // Assert that shared project resources can have the same ID as they're unique on tenant + ID not just ID
+        $collection5 = $this->client->call(Client::METHOD_POST, '/databases/' . $database2['body']['$id'] . '/collections', [
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $project2['body']['$id'],
+            'x-appwrite-key' => $key2['body']['secret']
+        ], [
+            'databaseId' => $database2['body']['$id'],
+            'collectionId' => $collection4['body']['$id'],
+            'name' => 'Amazing Collection',
+        ]);
+
+        $this->assertEquals(201, $collection5['headers']['status-code']);
+
+        // Assert that users across projects on shared tables can have the same email as they're unique on tenant + email not just email
+        $user1 = $this->client->call(Client::METHOD_POST, '/users', [
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $project2['body']['$id'],
+            'x-appwrite-key' => $key2['body']['secret']
+        ], [
+            'userId' => 'user',
+            'email' => 'test@appwrite.io',
+            'password' => 'password',
+            'name' => 'Test User',
+        ]);
+
+        $this->assertEquals(201, $user1['headers']['status-code']);
+
+        $user2 = $this->client->call(Client::METHOD_POST, '/users', [
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $project4['body']['$id'],
+            'x-appwrite-key' => $key4['body']['secret']
+        ], [
+            'userId' => 'user',
+            'email' => 'test@appwrite.io',
+            'password' => 'password',
+            'name' => 'Test User',
+        ]);
+
+        $this->assertEquals(201, $user2['headers']['status-code']);
+    }
 }

tests/e2e/Services/Projects/ProjectsCustomClientTest.php

@@ -1,18 +0,0 @@
-<?php
-
-namespace Tests\E2E\Services\Projects;
-
-use Tests\E2E\Scopes\ProjectCustom;
-use Tests\E2E\Scopes\Scope;
-use Tests\E2E\Scopes\SideClient;
-
-class ProjectsCustomClientTest extends Scope
-{
-    use ProjectCustom;
-    use SideClient;
-
-    public function testMock()
-    {
-        $this->assertEquals(true, true);
-    }
-}

tests/e2e/Services/Projects/ProjectsCustomServerTest.php

@@ -31,6 +31,10 @@ class ProjectsCustomServerTest extends Scope
 
         $this->assertEquals(201, $response['headers']['status-code']);
 
+        $response = $this->client->call(Client::METHOD_DELETE, '/proxy/rules/' . $response['body']['$id'], $headers);
+
+        $this->assertEquals(204, $response['headers']['status-code']);
+
         // prevent functions domain
         $functionsDomain = System::getEnv('_APP_DOMAIN_FUNCTIONS', '');
 

tests/e2e/Services/Realtime/RealtimeCustomClientTest.php

@@ -290,7 +290,6 @@ class RealtimeCustomClientTest extends Scope
 
         $this->assertEquals($name, $response['data']['payload']['name']);
 
-
         /**
          * Test Account Password Event
          */
@@ -376,6 +375,7 @@ class RealtimeCustomClientTest extends Scope
         $this->assertNotEmpty($response['data']);
         $this->assertCount(2, $response['data']['channels']);
         $this->assertArrayHasKey('timestamp', $response['data']);
+        $this->assertArrayNotHasKey('secret', $response['data']);
         $this->assertContains('account', $response['data']['channels']);
         $this->assertContains('account.' . $userId, $response['data']['channels']);
         $this->assertContains("users.{$userId}.verification.{$verificationId}.create", $response['data']['events']);
@@ -1317,7 +1317,7 @@ class RealtimeCustomClientTest extends Scope
             'x-appwrite-key' => $this->getProject()['apiKey']
         ]), []);
 
-        $this->assertEquals($response['headers']['status-code'], 200);
+        $this->assertEquals(200, $response['headers']['status-code']);
         $this->assertNotEmpty($response['body']['$id']);
 
         $execution = $this->client->call(Client::METHOD_POST, '/functions/' . $functionId . '/executions', array_merge([
@@ -1327,7 +1327,7 @@ class RealtimeCustomClientTest extends Scope
             'async' => true
         ]);
 
-        $this->assertEquals($execution['headers']['status-code'], 202);
+        $this->assertEquals(202, $execution['headers']['status-code']);
         $this->assertNotEmpty($execution['body']['$id']);
 
         $response = json_decode($client->receive(), true);

tests/e2e/Services/Storage/StorageBase.php

@@ -125,11 +125,11 @@ trait StorageBase
 
         /**
          * Failure
-         * Test for Chunk above 5MB
+         * Test for Chunk above 10MB
          */
         $source = __DIR__ . "/../../../resources/disk-a/large-file.mp4";
         $totalSize = \filesize($source);
-        $chunkSize = 6 * 1024 * 1024;
+        $chunkSize = 12 * 1024 * 1024;
         $handle = @fopen($source, "rb");
         $fileId = 'unique()';
         $mimeType = mime_content_type($source);

tests/e2e/Services/Webhooks/WebhooksCustomClientTest.php

@@ -44,7 +44,7 @@ class WebhooksCustomClientTest extends Scope
         $webhook = $this->getLastRequest();
         $signatureKey = $this->getProject()['signatureKey'];
         $payload = json_encode($webhook['data']);
-        $url     = $webhook['url'];
+        $url = $webhook['url'];
         $signatureExpected = base64_encode(hash_hmac('sha1', $url . $payload, $signatureKey, true));
 
         $this->assertEquals($webhook['method'], 'POST');
@@ -120,7 +120,7 @@ class WebhooksCustomClientTest extends Scope
         $webhook = $this->getLastRequest();
         $signatureKey = $this->getProject()['signatureKey'];
         $payload = json_encode($webhook['data']);
-        $url     = $webhook['url'];
+        $url = $webhook['url'];
         $signatureExpected = base64_encode(hash_hmac('sha1', $url . $payload, $signatureKey, true));
 
         $this->assertEquals($webhook['method'], 'POST');
@@ -174,7 +174,7 @@ class WebhooksCustomClientTest extends Scope
         $webhook = $this->getLastRequest();
         $signatureKey = $this->getProject()['signatureKey'];
         $payload = json_encode($webhook['data']);
-        $url     = $webhook['url'];
+        $url = $webhook['url'];
         $signatureExpected = base64_encode(hash_hmac('sha1', $url . $payload, $signatureKey, true));
 
         $this->assertEquals($webhook['method'], 'POST');
@@ -263,7 +263,7 @@ class WebhooksCustomClientTest extends Scope
         $webhook = $this->getLastRequest();
         $signatureKey = $this->getProject()['signatureKey'];
         $payload = json_encode($webhook['data']);
-        $url     = $webhook['url'];
+        $url = $webhook['url'];
         $signatureExpected = base64_encode(hash_hmac('sha1', $url . $payload, $signatureKey, true));
 
         $this->assertEquals($webhook['method'], 'POST');
@@ -349,7 +349,7 @@ class WebhooksCustomClientTest extends Scope
         $webhook = $this->getLastRequest();
         $signatureKey = $this->getProject()['signatureKey'];
         $payload = json_encode($webhook['data']);
-        $url     = $webhook['url'];
+        $url = $webhook['url'];
         $signatureExpected = base64_encode(hash_hmac('sha1', $url . $payload, $signatureKey, true));
 
         $this->assertEquals($webhook['method'], 'POST');
@@ -440,7 +440,7 @@ class WebhooksCustomClientTest extends Scope
         $webhook = $this->getLastRequest();
         $signatureKey = $this->getProject()['signatureKey'];
         $payload = json_encode($webhook['data']);
-        $url     = $webhook['url'];
+        $url = $webhook['url'];
         $signatureExpected = base64_encode(hash_hmac('sha1', $url . $payload, $signatureKey, true));
 
 
@@ -494,7 +494,7 @@ class WebhooksCustomClientTest extends Scope
         $webhook = $this->getLastRequest();
         $signatureKey = $this->getProject()['signatureKey'];
         $payload = json_encode($webhook['data']);
-        $url     = $webhook['url'];
+        $url = $webhook['url'];
         $signatureExpected = base64_encode(hash_hmac('sha1', $url . $payload, $signatureKey, true));
 
         $this->assertEquals($webhook['method'], 'POST');
@@ -549,7 +549,7 @@ class WebhooksCustomClientTest extends Scope
         $webhook = $this->getLastRequest();
         $signatureKey = $this->getProject()['signatureKey'];
         $payload = json_encode($webhook['data']);
-        $url     = $webhook['url'];
+        $url = $webhook['url'];
         $signatureExpected = base64_encode(hash_hmac('sha1', $url . $payload, $signatureKey, true));
 
         $this->assertEquals($webhook['method'], 'POST');
@@ -605,7 +605,7 @@ class WebhooksCustomClientTest extends Scope
         $webhook = $this->getLastRequest();
         $signatureKey = $this->getProject()['signatureKey'];
         $payload = json_encode($webhook['data']);
-        $url     = $webhook['url'];
+        $url = $webhook['url'];
         $signatureExpected = base64_encode(hash_hmac('sha1', $url . $payload, $signatureKey, true));
 
         $this->assertEquals($webhook['method'], 'POST');
@@ -661,7 +661,7 @@ class WebhooksCustomClientTest extends Scope
         $webhook = $this->getLastRequest();
         $signatureKey = $this->getProject()['signatureKey'];
         $payload = json_encode($webhook['data']);
-        $url     = $webhook['url'];
+        $url = $webhook['url'];
         $signatureExpected = base64_encode(hash_hmac('sha1', $url . $payload, $signatureKey, true));
 
         $this->assertEquals($webhook['method'], 'POST');
@@ -719,7 +719,7 @@ class WebhooksCustomClientTest extends Scope
         $webhook = $this->getLastRequest();
         $signatureKey = $this->getProject()['signatureKey'];
         $payload = json_encode($webhook['data']);
-        $url     = $webhook['url'];
+        $url = $webhook['url'];
         $signatureExpected = base64_encode(hash_hmac('sha1', $url . $payload, $signatureKey, true));
 
         $this->assertEquals($webhook['method'], 'POST');
@@ -774,7 +774,7 @@ class WebhooksCustomClientTest extends Scope
         $webhook = $this->getLastRequest();
         $signatureKey = $this->getProject()['signatureKey'];
         $payload = json_encode($webhook['data']);
-        $url     = $webhook['url'];
+        $url = $webhook['url'];
         $signatureExpected = base64_encode(hash_hmac('sha1', $url . $payload, $signatureKey, true));
 
         $this->assertEquals($webhook['method'], 'POST');
@@ -833,7 +833,7 @@ class WebhooksCustomClientTest extends Scope
         $webhook = $this->getLastRequest();
         $signatureKey = $this->getProject()['signatureKey'];
         $payload = json_encode($webhook['data']);
-        $url     = $webhook['url'];
+        $url = $webhook['url'];
         $signatureExpected = base64_encode(hash_hmac('sha1', $url . $payload, $signatureKey, true));
 
         $this->assertEquals($webhook['method'], 'POST');
@@ -891,7 +891,7 @@ class WebhooksCustomClientTest extends Scope
         $webhook = $this->getLastRequest();
         $signatureKey = $this->getProject()['signatureKey'];
         $payload = json_encode($webhook['data']);
-        $url     = $webhook['url'];
+        $url = $webhook['url'];
         $signatureExpected = base64_encode(hash_hmac('sha1', $url . $payload, $signatureKey, true));
 
         $this->assertEquals($webhook['method'], 'POST');

tests/resources/docker/docker-compose.yml

@@ -1,5 +1,3 @@
-version: '3'
-
 services:
   traefik:
     image: traefik:2.2

tests/unit/Docker/ComposeTest.php

@@ -21,11 +21,6 @@ class ComposeTest extends TestCase
         $this->object = new Compose($data);
     }
 
-    public function testVersion(): void
-    {
-        $this->assertEquals('3', $this->object->getVersion());
-    }
-
     public function testServices(): void
     {
         $this->assertCount(15, $this->object->getServices());

tests/unit/Event/EventTest.php

@@ -20,7 +20,7 @@ class EventTest extends TestCase
 
     public function setUp(): void
     {
-        $fallbackForRedis = URL::unparse([
+        $fallbackForRedis = 'redis_main=' . URL::unparse([
             'scheme' => 'redis',
             'host' => System::getEnv('_APP_REDIS_HOST', 'redis'),
             'port' => System::getEnv('_APP_REDIS_PORT', '6379'),
@@ -30,7 +30,7 @@ class EventTest extends TestCase
 
         $dsn = System::getEnv('_APP_CONNECTIONS_QUEUE', $fallbackForRedis);
         $dsn = explode('=', $dsn);
-        $dsn = $dsn[0] ?? '';
+        $dsn = $dsn[1] ?? '';
         $dsn = new DSN($dsn);
         $connection = new Queue\Connection\Redis($dsn->getHost(), $dsn->getPort());
         $this->queue = 'v1-tests' . uniqid();

tests/unit/Usage/StatsTest.php

@@ -19,7 +19,7 @@ class StatsTest extends TestCase
 
     public function setUp(): void
     {
-        $env =  System::getEnv('_APP_CONNECTIONS_QUEUE', AppwriteURL::unparse([
+        $env = System::getEnv('_APP_CONNECTIONS_QUEUE', 'redis_main=' . AppwriteURL::unparse([
             'scheme' => 'redis',
             'host' => System::getEnv('_APP_REDIS_HOST', 'redis'),
             'port' => System::getEnv('_APP_REDIS_PORT', '6379'),