Merge pull request #3741 from appwrite/refactor-guest-rate-limits

Refactor guest rate limits
2024-06-02 19:04:49 +12:00 · 2022-08-31 10:41:23 +02:00 · 2022-08-31 10:41:23 +02:00 · 5a58312b53
parent 054516d5c5 7ea521665c
commit 5a58312b53
7 changed files with 324 additions and 19 deletions
--- a/app/controllers/api/databases.php
+++ b/app/controllers/api/databases.php
@ -1813,8 +1813,9 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents')
    ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
    ->label('usage.metric', 'documents.{scope}.requests.create')
    ->label('usage.params', ['databaseId:{request.databaseId}', 'collectionId:{request.collectionId}'])
-    ->label('abuse-limit', 120)
-    ->label('abuse-time', 60)
+    ->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}')
+    ->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT * 2)
+    ->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT)
    ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
    ->label('sdk.namespace', 'databases')
    ->label('sdk.method', 'createDocument')
@ -2192,8 +2193,9 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum
    ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}/document/{response.$id}')
    ->label('usage.metric', 'documents.{scope}.requests.update')
    ->label('usage.params', ['databaseId:{request.databaseId}', 'collectionId:{request.collectionId}'])
-    ->label('abuse-limit', 60)
-    ->label('abuse-time', 60)
+    ->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}')
+    ->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT * 2)
+    ->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT)
    ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
    ->label('sdk.namespace', 'databases')
    ->label('sdk.method', 'updateDocument')
@ -2323,8 +2325,9 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu
    ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}/document/{request.documentId}')
    ->label('usage.metric', 'documents.{scope}.requests.delete')
    ->label('usage.params', ['databaseId:{request.databaseId}', 'collectionId:{request.collectionId}'])
-    ->label('abuse-limit', 60)
-    ->label('abuse-time', 60)
+    ->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}')
+    ->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT)
+    ->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT)
    ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
    ->label('sdk.namespace', 'databases')
    ->label('sdk.method', 'deleteDocument')
--- a/app/controllers/api/functions.php
+++ b/app/controllers/api/functions.php
@ -931,8 +931,9 @@ App::post('/v1/functions/:functionId/executions')
    ->label('sdk.response.code', Response::STATUS_CODE_CREATED)
    ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
    ->label('sdk.response.model', Response::MODEL_EXECUTION)
-    ->label('abuse-limit', 60)
-    ->label('abuse-time', 60)
+    ->label('abuse-key', 'ip:{ip},userId:{userId}')
+    ->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT)
+    ->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT)
    ->param('functionId', '', new UID(), 'Function ID.')
    ->param('data', '', new Text(8192), 'String of custom data to send to function.', true)
    ->param('async', true, new Boolean(), 'Execute code asynchronously. Default value is true.', true)
--- a/app/controllers/api/storage.php
+++ b/app/controllers/api/storage.php
@ -327,8 +327,9 @@ App::post('/v1/storage/buckets/:bucketId/files')
    ->label('audits.resource', 'files/{response.$id}')
    ->label('usage.metric', 'files.{scope}.requests.create')
    ->label('usage.params', ['bucketId:{request.bucketId}'])
-    ->label('abuse-limit', 60)
-    ->label('abuse-time', 60)
+    ->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}')
+    ->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT)
+    ->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT)
    ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
    ->label('sdk.namespace', 'storage')
    ->label('sdk.method', 'createFile')
@ -1207,8 +1208,9 @@ App::put('/v1/storage/buckets/:bucketId/files/:fileId')
    ->label('audits.resource', 'files/{response.$id}')
    ->label('usage.metric', 'files.{scope}.requests.update')
    ->label('usage.params', ['bucketId:{request.bucketId}'])
-    ->label('abuse-limit', 60)
-    ->label('abuse-time', 60)
+    ->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}')
+    ->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT)
+    ->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT)
    ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
    ->label('sdk.namespace', 'storage')
    ->label('sdk.method', 'updateFile')
@ -1308,8 +1310,9 @@ App::delete('/v1/storage/buckets/:bucketId/files/:fileId')
    ->label('audits.resource', 'file/{request.fileId}')
    ->label('usage.metric', 'files.{scope}.requests.delete')
    ->label('usage.params', ['bucketId:{request.bucketId}'])
-    ->label('abuse-limit', 60)
-    ->label('abuse-time', 60)
+    ->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}')
+    ->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT)
+    ->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT)
    ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
    ->label('sdk.namespace', 'storage')
    ->label('sdk.method', 'deleteFile')
--- a/app/controllers/shared/api.php
+++ b/app/controllers/shared/api.php
@ -83,7 +83,8 @@ App::init()
            ->setParam('{userId}', $user->getId())
            ->setParam('{userAgent}', $request->getUserAgent(''))
            ->setParam('{ip}', $request->getIP())
-            ->setParam('{url}', $request->getHostname() . $route->getPath());
+            ->setParam('{url}', $request->getHostname() . $route->getPath())
+            ->setParam('{method}', $request->getMethod());
            $timeLimitArray[] = $timeLimit;
        }

@ -114,11 +115,14 @@ App::init()
                ;
            }

+            $enabled = App::getEnv('_APP_OPTIONS_ABUSE', 'enabled') !== 'disabled';
+
            if (
-                (App::getEnv('_APP_OPTIONS_ABUSE', 'enabled') !== 'disabled' // Route is rate-limited
-                    && $abuse->check()) // Abuse is not disabled
-                && (!$isAppUser && !$isPrivilegedUser)
-            ) { // User is not an admin or API key
+                $enabled                // Abuse is enabled
+                && !$isAppUser          // User is not API key
+                && !$isPrivilegedUser   // User is not an admin
+                && $abuse->check()      // Route is rate-limited
+            ) {
                throw new Exception(Exception::GENERAL_RATE_LIMIT_EXCEEDED);
            }
        }
--- a/app/init.php
+++ b/app/init.php
@ -90,6 +90,8 @@ const APP_LIMIT_COMPRESSION = 20000000; //20MB
 const APP_LIMIT_ARRAY_PARAMS_SIZE = 100; // Default maximum of how many elements can there be in API parameter that expects array value
 const APP_LIMIT_ARRAY_ELEMENT_SIZE = 4096; // Default maximum length of element in array parameter represented by maximum URL length.
 const APP_LIMIT_SUBQUERY = 1000;
+const APP_LIMIT_WRITE_RATE_DEFAULT = 60; // Default maximum write rate per rate period
+const APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT = 60; // Default maximum write rate period in seconds
 const APP_KEY_ACCCESS = 24 * 60 * 60; // 24 hours
 const APP_CACHE_BUSTER = 402;
 const APP_VERSION_STABLE = '0.15.3';
--- a/tests/e2e/General/AbuseTest.php
+++ b/tests/e2e/General/AbuseTest.php
@ -0,0 +1,287 @@
+<?php
+
+namespace Tests\E2E\General;
+
+use CURLFile;
+use Tests\E2E\Client;
+use Tests\E2E\Scopes\ProjectCustom;
+use Tests\E2E\Scopes\Scope;
+use Tests\E2E\Scopes\SideNone;
+use Utopia\App;
+use Utopia\Database\ID;
+use Utopia\Database\Permission;
+use Utopia\Database\Role;
+
+class AbuseTest extends Scope
+{
+    use ProjectCustom;
+    use SideNone;
+
+    protected function setUp(): void
+    {
+        parent::setUp();
+
+        if (App::getEnv('_APP_OPTIONS_ABUSE') === 'disabled') {
+            $this->markTestSkipped('Abuse is not enabled.');
+        }
+    }
+
+    public function testAbuseCreateDocument()
+    {
+        $data = $this->createCollection();
+        $databaseId = $data['databaseId'];
+        $collectionId = $data['collectionId'];
+        $max = 120;
+
+        for ($i = 0; $i <= $max + 1; $i++) {
+            $response = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections/' . $collectionId . '/documents', [
+                'content-type' => 'application/json',
+                'x-appwrite-project' => $this->getProject()['$id'],
+            ], [
+                'documentId' => ID::unique(),
+                'data' => [
+                    'title' => 'The Hulk ' . $i,
+                ],
+            ]);
+
+            if ($i < $max) {
+                $this->assertEquals(201, $response['headers']['status-code']);
+            } else {
+                $this->assertEquals(429, $response['headers']['status-code']);
+            }
+        }
+    }
+
+    public function testAbuseUpdateDocument()
+    {
+        $data = $this->createCollection();
+        $databaseId = $data['databaseId'];
+        $collectionId = $data['collectionId'];
+        $max = 120;
+
+        $document = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections/' . $collectionId . '/documents', [
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $this->getProject()['$id'],
+            'x-appwrite-key' => $this->getProject()['apiKey'],
+        ], [
+            'documentId' => ID::unique(),
+            'data' => [
+                'title' => 'The Hulk',
+            ],
+        ]);
+
+        $documentId = $document['body']['$id'];
+
+        for ($i = 0; $i <= $max + 1; $i++) {
+            $response = $this->client->call(Client::METHOD_PATCH, '/databases/' . $databaseId . '/collections/' . $collectionId . '/documents/' . $documentId, [
+                'content-type' => 'application/json',
+                'x-appwrite-project' => $this->getProject()['$id'],
+            ], [
+                'data' => [
+                    'title' => 'The Hulk ' . $i,
+                ],
+            ]);
+
+            if ($i < $max) {
+                $this->assertEquals(200, $response['headers']['status-code']);
+            } else {
+                $this->assertEquals(429, $response['headers']['status-code']);
+            }
+        }
+    }
+
+    public function testAbuseDeleteDocument()
+    {
+        $data = $this->createCollection();
+        $databaseId = $data['databaseId'];
+        $collectionId = $data['collectionId'];
+        $max = 60;
+
+        for ($i = 0; $i <= $max + 1; $i++) {
+            $document = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections/' . $collectionId . '/documents', [
+                'content-type' => 'application/json',
+                'x-appwrite-project' => $this->getProject()['$id'],
+                'x-appwrite-key' => $this->getProject()['apiKey'],
+            ], [
+                'documentId' => ID::unique(),
+                'data' => [
+                    'title' => 'The Hulk',
+                ],
+            ]);
+
+            $documentId = $document['body']['$id'];
+
+            $response = $this->client->call(Client::METHOD_DELETE, '/databases/' . $databaseId . '/collections/' . $collectionId . '/documents/' . $documentId, [
+                'content-type' => 'application/json',
+                'x-appwrite-project' => $this->getProject()['$id'],
+            ]);
+
+            if ($i < $max) {
+                $this->assertEquals(204, $response['headers']['status-code']);
+            } else {
+                $this->assertEquals(429, $response['headers']['status-code']);
+            }
+        }
+    }
+
+    public function testAbuseCreateFile()
+    {
+        $data = $this->createBucket();
+        $bucketId = $data['bucketId'];
+        $max = 60;
+
+        for ($i = 0; $i <= $max + 1; $i++) {
+            $response = $this->client->call(Client::METHOD_POST, '/storage/buckets/' . $bucketId . '/files', [
+                'content-type' => 'multipart/form-data',
+                'x-appwrite-project' => $this->getProject()['$id'],
+            ], [
+                'fileId' => ID::unique(),
+                'file' => new CURLFile(realpath(__DIR__ . '/../../resources/logo.png'), 'image/png', 'permissions.png'),
+            ]);
+
+            if ($i < $max) {
+                $this->assertEquals(201, $response['headers']['status-code']);
+            } else {
+                $this->assertEquals(429, $response['headers']['status-code']);
+            }
+        }
+    }
+
+    public function testAbuseUpdateFile()
+    {
+        $data = $this->createBucket();
+        $bucketId = $data['bucketId'];
+        $max = 60;
+
+        $response = $this->client->call(Client::METHOD_POST, '/storage/buckets/' . $bucketId . '/files', [
+            'content-type' => 'multipart/form-data',
+            'x-appwrite-project' => $this->getProject()['$id'],
+            'x-appwrite-key' => $this->getProject()['apiKey'],
+        ], [
+            'fileId' => ID::unique(),
+            'file' => new CURLFile(realpath(__DIR__ . '/../../resources/logo.png'), 'image/png', 'permissions.png'),
+        ]);
+
+        $fileId = $response['body']['$id'];
+
+        for ($i = 0; $i <= $max + 1; $i++) {
+            $response = $this->client->call(Client::METHOD_PUT, '/storage/buckets/' . $bucketId . '/files/' . $fileId, [
+                'content-type' => 'application/json',
+                'x-appwrite-project' => $this->getProject()['$id'],
+            ], [
+                'name' => 'permissions' . $i . '.png',
+            ]);
+
+            if ($i < $max) {
+                $this->assertEquals(200, $response['headers']['status-code']);
+            } else {
+                $this->assertEquals(429, $response['headers']['status-code']);
+            }
+        }
+    }
+
+    public function testAbuseDeleteFile()
+    {
+        $data = $this->createBucket();
+        $bucketId = $data['bucketId'];
+        $max = 60;
+
+        for ($i = 0; $i <= $max + 1; $i++) {
+            $response = $this->client->call(Client::METHOD_POST, '/storage/buckets/' . $bucketId . '/files', [
+                'content-type' => 'multipart/form-data',
+                'x-appwrite-project' => $this->getProject()['$id'],
+                'x-appwrite-key' => $this->getProject()['apiKey'],
+            ], [
+                'fileId' => ID::unique(),
+                'file' => new CURLFile(realpath(__DIR__ . '/../../resources/logo.png'), 'image/png', 'permissions.png'),
+            ]);
+
+            $fileId = $response['body']['$id'];
+
+            $response = $this->client->call(Client::METHOD_DELETE, '/storage/buckets/' . $bucketId . '/files/' . $fileId, [
+                'content-type' => 'application/json',
+                'x-appwrite-project' => $this->getProject()['$id'],
+            ]);
+
+            if ($i < $max) {
+                $this->assertEquals(204, $response['headers']['status-code']);
+            } else {
+                $this->assertEquals(429, $response['headers']['status-code']);
+            }
+        }
+    }
+
+    private function createCollection(): array
+    {
+        $database = $this->client->call(Client::METHOD_POST, '/databases', array_merge([
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $this->getProject()['$id'],
+            'x-appwrite-key' => $this->getProject()['apiKey']
+        ]), [
+            'databaseId' => ID::unique(),
+            'name' => 'AbuseDatabase',
+        ]);
+
+        $this->assertEquals(201, $database['headers']['status-code']);
+        $this->assertEquals('AbuseDatabase', $database['body']['name']);
+
+        $databaseId = $database['body']['$id'];
+
+        $movies = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections', [
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $this->getProject()['$id'],
+            'x-appwrite-key' => $this->getProject()['apiKey']
+        ], [
+            'collectionId' => ID::unique(),
+            'name' => 'Movies',
+            'permissions' => [
+                Permission::read(Role::any()),
+                Permission::create(Role::any()),
+                Permission::update(Role::any()),
+                Permission::delete(Role::any()),
+            ],
+        ]);
+
+        $collectionId = $movies['body']['$id'];
+
+        $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections/' . $collectionId . '/attributes/string', [
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $this->getProject()['$id'],
+            'x-appwrite-key' => $this->getProject()['apiKey']
+        ], [
+            'key' => 'title',
+            'size' => 256,
+            'required' => true,
+        ]);
+
+        sleep(1);
+
+        return [
+            'databaseId' => $databaseId,
+            'collectionId' => $collectionId,
+        ];
+    }
+
+    private function createBucket(): array
+    {
+        $bucket = $this->client->call(Client::METHOD_POST, '/storage/buckets', [
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $this->getProject()['$id'],
+            'x-appwrite-key' => $this->getProject()['apiKey'],
+        ], [
+            'bucketId' => ID::unique(),
+            'name' => 'Test Bucket',
+            'fileSecurity' => true,
+            'permissions' => [
+                Permission::read(Role::any()),
+                Permission::create(Role::any()),
+                Permission::update(Role::any()),
+                Permission::delete(Role::any()),
+            ],
+        ]);
+
+        return [
+            'bucketId' => $bucket['body']['$id'],
+        ];
+    }
+}
--- a/tests/e2e/General/UsageTest.php
+++ b/tests/e2e/General/UsageTest.php
@ -2,6 +2,7 @@

 namespace Tests\E2E\General;

+use Appwrite\Tests\Retry;
 use Tests\E2E\Client;
 use Tests\E2E\Scopes\ProjectCustom;
 use Tests\E2E\Scopes\Scope;
@ -25,6 +26,7 @@ class UsageTest extends Scope
        parent::setUp();
    }

+    #[Retry(count: 1)]
    public function testUsersStats(): array
    {
        $project = $this->getProject(true);
@ -92,6 +94,7 @@ class UsageTest extends Scope
    }

    /** @depends testUsersStats */
+    #[Retry(count: 1)]
    public function testStorageStats(array $data): array
    {
        $projectId = $data['projectId'];
@ -236,6 +239,7 @@ class UsageTest extends Scope
    }

    /** @depends testStorageStats */
+    #[Retry(count: 1)]
    public function testDatabaseStats(array $data): array
    {
        $headers = $data['headers'];
@ -436,6 +440,7 @@ class UsageTest extends Scope


    /** @depends testDatabaseStats */
+    #[Retry(count: 1)]
    public function testFunctionsStats(array $data): void
    {
        $headers = $data['headers'];