From e748bb20e4fbabeb599c70a609eabc0df9aad858 Mon Sep 17 00:00:00 2001 From: shimon Date: Thu, 15 Dec 2022 19:38:25 +0200 Subject: [PATCH] benchmarks --- bin/benchmark | 3 + phpbench.json | 17 ++ tests/benchmarks/Scopes/Scope.php | 14 ++ tests/benchmarks/Services/Databases/Base.php | 151 ++++++++++++++++++ .../Databases/DatabasesCustomClientBench.php | 10 ++ .../Databases/DatabasesCustomServerBench.php | 10 ++ tests/benchmarks/Services/Functions/Base.php | 113 +++++++++++++ .../Functions/FunctionsCustomClientBench.php | 10 ++ .../Functions/FunctionsCustomServerBench.php | 21 +++ tests/benchmarks/Services/Storage/Base.php | 116 ++++++++++++++ .../Storage/StorageCustomClientBench.php | 10 ++ .../Storage/StorageCustomServerBench.php | 10 ++ .../Services/Users/UserCustomServerBench.php | 94 +++++++++++ tests/benchmarks/http.js | 34 ---- tests/benchmarks/ws.js | 59 ------- 15 files changed, 579 insertions(+), 93 deletions(-) create mode 100644 bin/benchmark create mode 100644 phpbench.json create mode 100644 tests/benchmarks/Scopes/Scope.php create mode 100644 tests/benchmarks/Services/Databases/Base.php create mode 100644 tests/benchmarks/Services/Databases/DatabasesCustomClientBench.php create mode 100644 tests/benchmarks/Services/Databases/DatabasesCustomServerBench.php create mode 100644 tests/benchmarks/Services/Functions/Base.php create mode 100644 tests/benchmarks/Services/Functions/FunctionsCustomClientBench.php create mode 100644 tests/benchmarks/Services/Functions/FunctionsCustomServerBench.php create mode 100644 tests/benchmarks/Services/Storage/Base.php create mode 100644 tests/benchmarks/Services/Storage/StorageCustomClientBench.php create mode 100644 tests/benchmarks/Services/Storage/StorageCustomServerBench.php create mode 100644 tests/benchmarks/Services/Users/UserCustomServerBench.php delete mode 100644 tests/benchmarks/http.js delete mode 100644 tests/benchmarks/ws.js diff --git a/bin/benchmark b/bin/benchmark new file mode 100644 index 0000000000..de9bee100c --- /dev/null +++ b/bin/benchmark @@ -0,0 +1,3 @@ +#!/usr/bin/env sh + +/usr/src/code/vendor/bin/phpbench run --config /usr/src/code/phpbench.json --report appwrite $@ \ No newline at end of file diff --git a/phpbench.json b/phpbench.json new file mode 100644 index 0000000000..4ebdc080b4 --- /dev/null +++ b/phpbench.json @@ -0,0 +1,17 @@ +{ + "$schema": "./vendor/phpbench/phpbench/phpbench.schema.json", + "runner.path": "tests/benchmarks", + "runner.file_pattern": "*Bench.php", + "runner.bootstrap": "app/init.php", + "runner.revs": 1000, + "runner.iterations": 3, + "runner.retry_threshold": 5, + "runner.warmup": 1, + "report.generators": { + "appwrite": { + "extends": "aggregate", + "cols": ["benchmark", "subject", "set" ,"revs", "its", "worst", "best", "mean", "mode", "rstdev"], + "break": ["benchmark"] + } + } +} \ No newline at end of file diff --git a/tests/benchmarks/Scopes/Scope.php b/tests/benchmarks/Scopes/Scope.php new file mode 100644 index 0000000000..03b2d7f1ef --- /dev/null +++ b/tests/benchmarks/Scopes/Scope.php @@ -0,0 +1,14 @@ +client->call(Client::METHOD_POST, '/databases/' . static::$databaseId . '/collections/' . static::$collectionId . '/documents', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'documentId' => ID::unique(), + 'data' => [ + 'title' => 'The Matrix', + ], + 'permissions' => [ + Permission::read(Role::user($this->getUser()['$id'])), + Permission::write(Role::user($this->getUser()['$id'])), + ], + ]); + } + + #[ParamProviders(['provideCounts'])] + #[BeforeMethods(['createDatabase', 'createCollection', 'createDocuments'])] + public function benchDocumentReadList(array $params) + { + $this->client->call(Client::METHOD_GET, '/databases/' . static::$databaseId . '/collections/' . static::$collectionId . '/documents', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'queries' => ['limit(' . $params['documents'] . ')'], + ]); + } + + #[BeforeMethods(['createDatabase', 'createCollection', 'createDocuments'])] + public function benchDocumentRead() + { + $this->client->call(Client::METHOD_GET, '/databases/' . static::$databaseId . '/collections/' . static::$collectionId . '/documents/' . static::$documentId, array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + } + + #[BeforeMethods(['createDatabase', 'createCollection', 'createDocuments'])] + public function benchDocumentUpdate() + { + $this->client->call(Client::METHOD_PATCH, '/databases/' . static::$databaseId . '/collections/' . static::$collectionId . '/documents/' . static::$documentId, array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'data' => [ + 'title' => 'The Matrix Reloaded', + ], + ]); + } + + public function provideCounts(): array + { + return [ + '1 Document' => ['documents' => 1], + '10 Documents' => ['documents' => 10], + '100 Documents' => ['documents' => 100], + ]; + } + + public function createDatabase(array $params = []) + { + $database = $this->client->call(Client::METHOD_POST, '/databases', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'databaseId' => ID::unique(), + 'name' => 'Test Database' + ]); + static::$databaseId = $database['body']['$id']; + } + + public function createCollection(array $params = []) + { + $collection = $this->client->call(Client::METHOD_POST, '/databases/' . static::$databaseId . '/collections', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'collectionId' => ID::unique(), + 'name' => 'Movies', + 'documentSecurity' => true, + 'permissions' => [ + Permission::read(Role::user($this->getUser()['$id'])), + Permission::write(Role::user($this->getUser()['$id'])), + ], + ]); + static::$collectionId = $collection['body']['$id']; + + // Create attribute + $this->client->call(Client::METHOD_POST, '/databases/' . static::$databaseId . '/collections/' . static::$collectionId . '/attributes/string', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'key' => 'title', + 'size' => 256, + 'required' => true, + ]); + + // Wait for attribute to be ready + sleep(2); + } + + public function createDocuments(array $params = []) + { + $count = $params['documents'] ?? 1; + + for ($i = 0; $i < $count; $i++) { + $response = $this->client->call(Client::METHOD_POST, '/databases/' . static::$databaseId . '/collections/' . static::$collectionId . '/documents', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'documentId' => ID::unique(), + 'data' => [ + 'title' => 'Captain America' . $i, + ], + 'permissions' => [ + Permission::read(Role::user($this->getUser()['$id'])), + Permission::write(Role::user($this->getUser()['$id'])), + ] + ]); + + static::$documentId = $response['body']['$id']; + } + } +} diff --git a/tests/benchmarks/Services/Databases/DatabasesCustomClientBench.php b/tests/benchmarks/Services/Databases/DatabasesCustomClientBench.php new file mode 100644 index 0000000000..658ddfd98c --- /dev/null +++ b/tests/benchmarks/Services/Databases/DatabasesCustomClientBench.php @@ -0,0 +1,10 @@ +client->call(Client::METHOD_POST, '/functions/' . static::$functionId . '/executions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + } + + public function createFunction() + { + $response = $this->client->call(Client::METHOD_POST, '/functions', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'functionId' => ID::unique(), + 'name' => 'Test', + 'runtime' => 'php-8.0', + 'timeout' => 10, + 'execute' => [Role::users()->toString()] + ]); + static::$functionId = $response['body']['$id']; + } + + public function prepareDeployment() + { + $stdout = ''; + $stderr = ''; + + Console::execute( + 'cd ' . realpath(__DIR__ . "/../../resources/functions/php") . " && \ + tar --exclude code.tar.gz -czf code.tar.gz .", + '', + $stdout, + $stderr + ); + } + + public function createDeployment() + { + $code = realpath(__DIR__ . '/../../resources/functions/php/code.tar.gz'); + + $response = $this->client->call(Client::METHOD_POST, '/functions/' . static::$functionId . '/deployments', [ + 'content-type' => 'multipart/form-data', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'entrypoint' => 'index.php', + 'code' => new CURLFile( + $code, + 'application/x-gzip', + \basename($code) + ), + ]); + + static::$deploymentId = $response['body']['$id']; + + while (true) { + $response = $this->client->call(Client::METHOD_GET, '/functions/' . static::$functionId . '/deployments/' . static::$deploymentId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]); + + $status = $response['body']['status'] ?? ''; + + switch ($status) { + case '': + case 'processing': + case 'building': + usleep(200); + break; + case 'ready': + break 2; + case 'failed': + throw new \Exception('Failed to build function'); + } + } + + sleep(1); + } + + public function patchDeployment() + { + $this->client->call(Client::METHOD_PATCH, '/functions/' . static::$functionId . '/deployments/' . static::$deploymentId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], []); + } +} diff --git a/tests/benchmarks/Services/Functions/FunctionsCustomClientBench.php b/tests/benchmarks/Services/Functions/FunctionsCustomClientBench.php new file mode 100644 index 0000000000..ec5e0b8f29 --- /dev/null +++ b/tests/benchmarks/Services/Functions/FunctionsCustomClientBench.php @@ -0,0 +1,10 @@ +createDeployment(); + } +} diff --git a/tests/benchmarks/Services/Storage/Base.php b/tests/benchmarks/Services/Storage/Base.php new file mode 100644 index 0000000000..64a8ccaec5 --- /dev/null +++ b/tests/benchmarks/Services/Storage/Base.php @@ -0,0 +1,116 @@ +client->call(Client::METHOD_POST, '/storage/buckets/' . static::$bucketId . '/files', array_merge([ + 'content-type' => 'multipart/form-data', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'fileId' => ID::unique(), + 'permissions' => [ + Permission::read(Role::user($this->getUser()['$id'])), + Permission::write(Role::user($this->getUser()['$id'])), + ], + 'file' => new CURLFile(realpath(__DIR__ . '/../../resources/logo.png'), 'image/png', 'logo.png'), + ]); + } + + #[ParamProviders(['provideCounts'])] + #[BeforeMethods(['createBucket', 'createFiles'])] + public function benchFileReadList(array $params) + { + $this->client->call(Client::METHOD_GET, '/storage/buckets/' . static::$bucketId . '/files', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'queries' => ['limit(' . $params['files'] . ')'], + ]); + } + + #[BeforeMethods(['createBucket', 'createFiles'])] + public function benchFileRead() + { + $this->client->call(Client::METHOD_GET, '/storage/buckets/' . static::$bucketId . '/files/' . static::$fileId, array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + } + + #[BeforeMethods(['createBucket', 'createFiles'])] + public function benchFileUpdate() + { + $this->client->call(Client::METHOD_PUT, '/storage/buckets/' . static::$bucketId . '/files/' . static::$fileId, array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'name' => 'Updated name', + 'permissions' => [], + ]); + } + + public function provideCounts(): array + { + return [ + '10 Files' => ['files' => 10], + '100 Files' => ['files' => 100], + ]; + } + + public function createBucket(array $params = []) + { + // Create bucket + $bucket = $this->client->call(Client::METHOD_POST, '/storage/buckets', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'bucketId' => ID::unique(), + 'name' => 'Test Bucket', + 'permissions' => [ + Permission::read(Role::user($this->getUser()['$id'])), + Permission::write(Role::user($this->getUser()['$id'])), + ], + 'fileSecurity' => true + ]); + static::$bucketId = $bucket['body']['$id']; + } + + public function createFiles(array $params = []) + { + $count = $params['files'] ?? 1; + + // Create files + for ($i = 0; $i < $count; $i++) { + $response = $this->client->call(Client::METHOD_POST, '/storage/buckets/' . static::$bucketId . '/files', [ + 'content-type' => 'multipart/form-data', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'fileId' => ID::unique(), + 'file' => new CURLFile(realpath(__DIR__ . '/../../resources/logo.png'), 'image/png', 'logo.png'), + ]); + + static::$fileId = $response['body']['$id']; + } + } +} diff --git a/tests/benchmarks/Services/Storage/StorageCustomClientBench.php b/tests/benchmarks/Services/Storage/StorageCustomClientBench.php new file mode 100644 index 0000000000..585182c1da --- /dev/null +++ b/tests/benchmarks/Services/Storage/StorageCustomClientBench.php @@ -0,0 +1,10 @@ +client->call(Client::METHOD_POST, '/users', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'userId' => $id, + 'email' => 'test' . $id . '@example.com', + 'password' => 'password', + ]); + } + + #[ParamProviders(['provideCounts'])] + #[BeforeMethods(['createUsers'])] + public function benchUserReadList(array $params) + { + $this->client->call(Client::METHOD_GET, '/users', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'queries' => ['limit(' . $params['users'] . ')'], + ]); + } + + #[BeforeMethods(['createUsers'])] + public function benchUserRead() + { + $this->client->call(Client::METHOD_GET, '/users/' . static::$userId, array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + } + + #[BeforeMethods(['createUsers'])] + public function benchUserUpdate() + { + $this->client->call(Client::METHOD_PUT, '/users/' . static::$userId, array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'name' => 'New Name', + ]); + } + + public function createUsers(array $params = []) + { + $count = $params['documents'] ?? 1; + + for ($i = 0; $i < $count; $i++) { + $id = ID::unique(); + + $response = $this->client->call(Client::METHOD_POST, '/users', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'userId' => $id, + 'email' => 'test' . $id . '@example.com', + 'password' => 'password', + ]); + + static::$userId = $response['body']['$id']; + } + } + + public function provideCounts(): array + { + return [ + '1 User' => ['users' => 1], + '10 Users' => ['users' => 10], + '100 Users' => ['users' => 100], + ]; + } +} diff --git a/tests/benchmarks/http.js b/tests/benchmarks/http.js deleted file mode 100644 index 799c8fb23c..0000000000 --- a/tests/benchmarks/http.js +++ /dev/null @@ -1,34 +0,0 @@ -import http from 'k6/http'; -import { check } from 'k6'; -import { Counter } from 'k6/metrics'; - -// A simple counter for http requests -export const requests = new Counter('http_reqs'); - -// you can specify stages of your test (ramp up/down patterns) through the options object -// target is the number of VUs you are aiming for - -export const options = { - stages: [ - { target: 50, duration: '1m' }, - // { target: 15, duration: '1m' }, - // { target: 0, duration: '1m' }, - ], - thresholds: { - requests: ['count < 100'], - }, -}; - -export default function () { - const config = { - headers: { - 'X-Appwrite-Key': '24356eb021863f81eb7dd77c7750304d0464e141cad6e9a8befa1f7d2b066fde190df3dab1e8d2639dbb82ee848da30501424923f4cd80d887ee40ad77ded62763ee489448523f6e39667f290f9a54b2ab8fad131a0bc985e6c0f760015f7f3411e40626c75646bb19d2bb2f7bf2f63130918220a206758cbc48845fd725a695', - 'X-Appwrite-Project': '60479fe35d95d' - }} - - const resDb = http.get('http://localhost:9501/', config); - - check(resDb, { - 'status is 200': (r) => r.status === 200, - }); -} \ No newline at end of file diff --git a/tests/benchmarks/ws.js b/tests/benchmarks/ws.js deleted file mode 100644 index 916458856f..0000000000 --- a/tests/benchmarks/ws.js +++ /dev/null @@ -1,59 +0,0 @@ -// k6 run tests/benchmarks/ws.js - -import { URL } from 'https://jslib.k6.io/url/1.0.0/index.js'; -import ws from 'k6/ws'; -import { check } from 'k6'; - -export let options = { - stages: [ - { - duration: '10s', - target: 500 - }, - { - duration: '1m', - target: 500 - }, - ], -} - -export default function () { - // const url = new URL('wss://appwrite-realtime.monitor-api.com/v1/realtime'); - // url.searchParams.append('project', '604249e6b1a9f'); - const url = new URL('ws://localhost/v1/realtime'); - url.searchParams.append('project', 'console'); - url.searchParams.append('channels[]', 'files'); - - const res = ws.connect(url.toString(), function (socket) { - let connection = false; - let checked = false; - let payload = null; - socket.on('open', () => { - connection = true; - }); - - socket.on('message', (data) => { - payload = data; - checked = true; - }); - - socket.setTimeout(function () { - check(payload, { - 'connection opened': (r) => connection, - 'message received': (r) => checked, - 'channels are right': (r) => r === JSON.stringify({ - "type": "connected", - "data": { - "channels": [ - "files" - ], - "user": null - } - }) - }) - socket.close(); - }, 5000); - }); - - check(res, { 'status is 101': (r) => r && r.status === 101 }); -} \ No newline at end of file