diff --git a/Dockerfile b/Dockerfile index 12ac410b7..68b53747b 100755 --- a/Dockerfile +++ b/Dockerfile @@ -238,7 +238,6 @@ RUN chmod +x /usr/local/bin/doctor && \ chmod +x /usr/local/bin/worker-deletes && \ chmod +x /usr/local/bin/worker-functions && \ chmod +x /usr/local/bin/worker-mails && \ - chmod +x /usr/local/bin/worker-usage && \ chmod +x /usr/local/bin/worker-webhooks # Letsencrypt Permissions diff --git a/app/controllers/api/database.php b/app/controllers/api/database.php index 83c2120be..a4f383ee7 100644 --- a/app/controllers/api/database.php +++ b/app/controllers/api/database.php @@ -107,7 +107,7 @@ $attributesCallback = function ($attribute, $response, $dbForExternal, $database ]); $database - ->setParam('type', CREATE_TYPE_ATTRIBUTE) + ->setParam('type', DATABASE_TYPE_CREATE_ATTRIBUTE) ->setParam('document', $attribute) ; @@ -813,25 +813,24 @@ App::delete('/v1/database/collections/:collectionId/attributes/:attributeId') throw new Exception('Collection not found', 404); } - // $attributes = $collection->getAttributes(); + /** @var Document[] $attributes */ + $attributes = $collection->getAttribute('attributes'); - // Search for attribute - // $attributeIndex = array_search($attributeId, array_column($attributes, '$id')); - $attribute = $collection->find('$id', $attributeId, 'attributes'); + // find attribute in collection + $attribute = null; + foreach ($attributes as $a) { + if ($a->getId() === $attributeId) { + $attribute = $a->setAttribute('$collection', $collectionId); // set the collectionId + break; // break once attribute is found + } + } - if (empty($attribute) || !$attribute instanceof Document) { + if (\is_null($attribute)) { throw new Exception('Attribute not found', 404); } - // $attribute = new Document([\array_merge($attributes[$attributeIndex], [ - // 'collectionId' => $collectionId, - // ])]); - - // $type = $attribute->getAttribute('type', ''); - // $format = $attribute->getAttribute('format', ''); - $database - ->setParam('type', DELETE_TYPE_ATTRIBUTE) + ->setParam('type', DATABASE_TYPE_DELETE_ATTRIBUTE) ->setParam('document', $attribute) ; @@ -869,7 +868,7 @@ App::post('/v1/database/collections/:collectionId/indexes') ->inject('dbForExternal') ->inject('database') ->inject('audits') - ->action(function ($collectionId, $id, $type, $attributes, $orders, $response, $dbForExternal, $database, $audits) { + ->action(function ($collectionId, $indexId, $type, $attributes, $orders, $response, $dbForExternal, $database, $audits) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForExternal */ /** @var Appwrite\Event\Event $database */ @@ -905,7 +904,7 @@ App::post('/v1/database/collections/:collectionId/indexes') $lengths[$key] = ($attributeType === Database::VAR_STRING) ? $attributeSize : null; } - $success = $dbForExternal->addIndexInQueue($collectionId, $id, $type, $attributes, $lengths, $orders); + $success = $dbForExternal->addIndexInQueue($collectionId, $indexId, $type, $attributes, $lengths, $orders); // Database->createIndex() does not return a document // So we need to create one for the response @@ -913,7 +912,7 @@ App::post('/v1/database/collections/:collectionId/indexes') // TODO@kodumbeats should $lengths be a part of the response model? $index = new Document([ '$collection' => $collectionId, - '$id' => $id, + '$id' => $indexId, 'type' => $type, 'attributes' => $attributes, 'lengths' => $lengths, @@ -921,7 +920,7 @@ App::post('/v1/database/collections/:collectionId/indexes') ]); $database - ->setParam('type', CREATE_TYPE_INDEX) + ->setParam('type', DATABASE_TYPE_CREATE_INDEX) ->setParam('document', $index) ; @@ -1046,21 +1045,24 @@ App::delete('/v1/database/collections/:collectionId/indexes/:indexId') throw new Exception('Collection not found', 404); } + /** @var Document[] $indexes */ $indexes = $collection->getAttribute('indexes'); - // // Search for index - $indexIndex = array_search($indexId, array_column($indexes, '$id')); + // find attribute in collection + $index= null; + foreach ($indexes as $i) { + if ($i->getId() === $indexId) { + $index = $i->setAttribute('$collection', $collectionId); // set the collectionId + break; // break once index is found + } + } - if ($indexIndex === false) { + if (\is_null($index)) { throw new Exception('Index not found', 404); } - $index = new Document([\array_merge($indexes[$indexIndex], [ - 'collectionId' => $collectionId, - ])]); - $database - ->setParam('type', DELETE_TYPE_INDEX) + ->setParam('type', DATABASE_TYPE_DELETE_INDEX) ->setParam('document', $index) ; diff --git a/app/controllers/general.php b/app/controllers/general.php index 91b96edb8..fa8f7b9cb 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -53,7 +53,6 @@ App::init(function ($utopia, $request, $response, $console, $project, $dbForCons 'domain' => $domain->get(), ]); $certificate = $dbForConsole->createDocument('certificates', $certificate); - Authorization::enable(); Console::info('Issuing a TLS certificate for the master domain (' . $domain->get() . ') in a few seconds...'); @@ -63,10 +62,11 @@ App::init(function ($utopia, $request, $response, $console, $project, $dbForCons 'validateTarget' => false, 'validateCNAME' => false, ]); - } else { - Authorization::enable(); // ensure authorization is reenabled } + $domains[$domain->get()] = true; + + Authorization::reset(); // ensure authorization is re-enabled } Config::setParam('domains', $domains); } diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 8a9256720..674a5004b 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -18,7 +18,7 @@ App::init(function ($utopia, $request, $response, $project, $user, $register, $e /** @var Utopia\Registry\Registry $register */ /** @var Appwrite\Event\Event $events */ /** @var Appwrite\Event\Event $audits */ - /** @var Appwrite\Event\Event $usage */ + /** @var Appwrite\Stats\Stats $usage */ /** @var Appwrite\Event\Event $deletes */ /** @var Appwrite\Event\Event $database */ /** @var Appwrite\Event\Event $functions */ @@ -162,14 +162,14 @@ App::init(function ($utopia, $request, $project) { }, ['utopia', 'request', 'project'], 'auth'); -App::shutdown(function ($utopia, $request, $response, $project, $events, $audits, $usage, $deletes, $database, $mode) { +App::shutdown(function ($utopia, $request, $response, $project, $register, $events, $audits, $usage, $deletes, $database, $mode) { /** @var Utopia\App $utopia */ /** @var Utopia\Swoole\Request $request */ /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Document $project */ /** @var Appwrite\Event\Event $events */ /** @var Appwrite\Event\Event $audits */ - /** @var Appwrite\Event\Event $usage */ + /** @var Appwrite\Stats\Stats $usage */ /** @var Appwrite\Event\Event $deletes */ /** @var Appwrite\Event\Event $database */ /** @var Appwrite\Event\Event $functions */ @@ -215,8 +215,8 @@ App::shutdown(function ($utopia, $request, $response, $project, $events, $audits $usage ->setParam('networkRequestSize', $request->getSize() + $usage->getParam('storage')) ->setParam('networkResponseSize', $response->getSize()) - ->trigger() + ->submit() ; } -}, ['utopia', 'request', 'response', 'project', 'events', 'audits', 'usage', 'deletes', 'database', 'mode'], 'api'); +}, ['utopia', 'request', 'response', 'project', 'register', 'events', 'audits', 'usage', 'deletes', 'database', 'mode'], 'api'); \ No newline at end of file diff --git a/app/init.php b/app/init.php index 72b3dedc7..46dafebac 100644 --- a/app/init.php +++ b/app/init.php @@ -29,6 +29,7 @@ use Appwrite\Network\Validator\Email; use Appwrite\Network\Validator\IP; use Appwrite\Network\Validator\URL; use Appwrite\OpenSSL\OpenSSL; +use Appwrite\Stats\Stats; use Utopia\App; use Utopia\View; use Utopia\Config\Config; @@ -76,18 +77,18 @@ const APP_SOCIAL_DISCORD = 'https://appwrite.io/discord'; const APP_SOCIAL_DISCORD_CHANNEL = '564160730845151244'; const APP_SOCIAL_DEV = 'https://dev.to/appwrite'; const APP_SOCIAL_STACKSHARE = 'https://stackshare.io/appwrite'; -// Creation Types -const CREATE_TYPE_ATTRIBUTE = 'newAttribute'; -const CREATE_TYPE_INDEX = 'newIndex'; -// Deletion Types -const DELETE_TYPE_ATTRIBUTE = 'attribute'; -const DELETE_TYPE_INDEX = 'index'; +// Database Worker Types +const DATABASE_TYPE_CREATE_ATTRIBUTE = 'createAttribute'; +const DATABASE_TYPE_CREATE_INDEX = 'createIndex'; +const DATABASE_TYPE_DELETE_ATTRIBUTE = 'deleteAttribute'; +const DATABASE_TYPE_DELETE_INDEX = 'deleteIndex'; +// Deletes Worker Types const DELETE_TYPE_DOCUMENT = 'document'; const DELETE_TYPE_EXECUTIONS = 'executions'; const DELETE_TYPE_AUDIT = 'audit'; const DELETE_TYPE_ABUSE = 'abuse'; const DELETE_TYPE_CERTIFICATES = 'certificates'; -// Mail Types +// Mail Worker Types const MAIL_TYPE_VERIFICATION = 'verification'; const MAIL_TYPE_RECOVERY = 'recovery'; const MAIL_TYPE_INVITATION = 'invitation'; @@ -291,6 +292,7 @@ $register->set('statsd', function () { // Register DB connection return $statsd; }); + $register->set('smtp', function () { $mail = new PHPMailer(true); @@ -421,7 +423,7 @@ App::setResource('audits', function($register) { }, ['register']); App::setResource('usage', function($register) { - return new Event(Event::USAGE_QUEUE_NAME, Event::USAGE_CLASS_NAME); + return new Stats($register->get('statsd')); }, ['register']); App::setResource('mails', function($register) { diff --git a/app/views/console/database/collection.phtml b/app/views/console/database/collection.phtml index c96401fa9..43254c459 100644 --- a/app/views/console/database/collection.phtml +++ b/app/views/console/database/collection.phtml @@ -16,8 +16,8 @@
- - + +

JSON View

@@ -52,7 +52,7 @@
documents found
- +
diff --git a/app/views/console/database/document.phtml b/app/views/console/database/document.phtml index bdc87aed8..1d0a6089d 100644 --- a/app/views/console/database/document.phtml +++ b/app/views/console/database/document.phtml @@ -68,16 +68,16 @@
- +

Permissions

- + -
Add * for wildcard access
+
Add 'role:all' for wildcard access
-
Add * for wildcard access
+
Add 'role:all' for wildcard access
diff --git a/app/views/console/functions/function.phtml b/app/views/console/functions/function.phtml index 0ec16db90..99c160406 100644 --- a/app/views/console/functions/function.phtml +++ b/app/views/console/functions/function.phtml @@ -453,7 +453,7 @@ $usageStatsEnabled = $this->getParam('usageStatsEnabled', true); -
Add * for wildcard access
+
Add 'role:all' for wildcard access
diff --git a/app/views/console/storage/index.phtml b/app/views/console/storage/index.phtml index 163f2ae6d..4cbfc29d0 100644 --- a/app/views/console/storage/index.phtml +++ b/app/views/console/storage/index.phtml @@ -107,11 +107,11 @@ $fileLimitHuman = $this->getParam('fileLimitHuman', 0); -
Add * for wildcard access
+
Add 'role:all' for wildcard access
-
Add * for wildcard access
+
Add 'role:all' for wildcard access
getParam('fileLimitHuman', 0); -
Add * for wildcard access
+
Add 'role:all' for wildcard access
-
Add * for wildcard access
+
Add 'role:all' for wildcard access
  diff --git a/app/workers/database.php b/app/workers/database.php index 60c060ee7..54a7d1933 100644 --- a/app/workers/database.php +++ b/app/workers/database.php @@ -26,22 +26,22 @@ class DatabaseV1 extends Worker Authorization::disable(); switch (strval($type)) { - case CREATE_TYPE_ATTRIBUTE: + case DATABASE_TYPE_CREATE_ATTRIBUTE: $attribute = $this->args['document'] ?? ''; $attribute = new Document($attribute); $this->createAttribute($attribute, $projectId); break; - case DELETE_TYPE_ATTRIBUTE: + case DATABASE_TYPE_DELETE_ATTRIBUTE: $attribute = $this->args['document'] ?? ''; $attribute = new Document($attribute); $this->deleteAttribute($attribute, $projectId); break; - case CREATE_TYPE_INDEX: + case DATABASE_TYPE_CREATE_INDEX: $index = $this->args['document'] ?? ''; $index = new Document($index); $this->createIndex($index, $projectId); break; - case DELETE_TYPE_INDEX: + case DATABASE_TYPE_DELETE_INDEX: $index = $this->args['document'] ?? ''; $index = new Document($index); $this->deleteIndex($index, $projectId); diff --git a/app/workers/functions.php b/app/workers/functions.php index 9e002dd70..bed0bda13 100644 --- a/app/workers/functions.php +++ b/app/workers/functions.php @@ -2,6 +2,7 @@ use Appwrite\Event\Event; use Appwrite\Resque\Worker; +use Appwrite\Stats\Stats; use Appwrite\Utopia\Response\Model\Execution; use Cron\CronExpression; use Swoole\Runtime; @@ -134,8 +135,6 @@ class FunctionsV1 extends Worker public function run(): void { - global $register; - $projectId = $this->args['projectId'] ?? ''; $functionId = $this->args['functionId'] ?? ''; $webhooks = $this->args['webhooks'] ?? []; @@ -279,7 +278,7 @@ class FunctionsV1 extends Worker */ public function execute(string $trigger, string $projectId, string $executionId, Database $database, Document $function, string $event = '', string $eventData = '', string $data = '', array $webhooks = [], string $userId = '', string $jwt = ''): void { - global $list; + global $list, $register; $runtimes = Config::getParam('runtimes'); @@ -477,21 +476,22 @@ class FunctionsV1 extends Worker ->setParam('eventData', $execution->getArrayCopy(array_keys($executionModel->getRules()))); $executionUpdate->trigger(); - - $usage = new Event('v1-usage', 'UsageV1'); - - $usage - ->setParam('projectId', $projectId) - ->setParam('functionId', $function->getId()) - ->setParam('functionExecution', 1) - ->setParam('functionStatus', $functionStatus) - ->setParam('functionExecutionTime', $executionTime * 1000) // ms - ->setParam('networkRequestSize', 0) - ->setParam('networkResponseSize', 0) - ; if(App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { - $usage->trigger(); + $statsd = $register->get('statsd'); + + $usage = new Stats($statsd); + + $usage + ->setParam('projectId', $projectId) + ->setParam('functionId', $function->getId()) + ->setParam('functionExecution', 1) + ->setParam('functionStatus', $functionStatus) + ->setParam('functionExecutionTime', $executionTime * 1000) // ms + ->setParam('networkRequestSize', 0) + ->setParam('networkResponseSize', 0) + ->submit() + ; } $this->cleanup(); diff --git a/app/workers/usage.php b/app/workers/usage.php deleted file mode 100644 index b5a3f885a..000000000 --- a/app/workers/usage.php +++ /dev/null @@ -1,71 +0,0 @@ -get('statsd', true); - - $projectId = $this->args['projectId'] ?? ''; - - $storage = $this->args['storage'] ?? 0; - - $networkRequestSize = $this->args['networkRequestSize'] ?? 0; - $networkResponseSize = $this->args['networkResponseSize'] ?? 0; - - $httpMethod = $this->args['httpMethod'] ?? ''; - $httpRequest = $this->args['httpRequest'] ?? 0; - - $functionId = $this->args['functionId'] ?? ''; - $functionExecution = $this->args['functionExecution'] ?? 0; - $functionExecutionTime = $this->args['functionExecutionTime'] ?? 0; - $functionStatus = $this->args['functionStatus'] ?? ''; - - $tags = ",project={$projectId},version=".App::getEnv('_APP_VERSION', 'UNKNOWN'); - - // the global namespace is prepended to every key (optional) - $statsd->setNamespace('appwrite.usage'); - - if($httpRequest >= 1) { - $statsd->increment('requests.all'.$tags.',method='.\strtolower($httpMethod)); - } - - if($functionExecution >= 1) { - $statsd->increment('executions.all'.$tags.',functionId='.$functionId.',functionStatus='.$functionStatus); - $statsd->count('executions.time'.$tags.',functionId='.$functionId, $functionExecutionTime); - } - - $statsd->count('network.inbound'.$tags, $networkRequestSize); - $statsd->count('network.outbound'.$tags, $networkResponseSize); - $statsd->count('network.all'.$tags, $networkRequestSize + $networkResponseSize); - - if($storage >= 1) { - $statsd->count('storage.all'.$tags, $storage); - } - } - - public function shutdown(): void - { - } -} \ No newline at end of file diff --git a/bin/worker-database b/bin/worker-database index 97e067d0d..3dfbeaaad 100644 --- a/bin/worker-database +++ b/bin/worker-database @@ -7,4 +7,4 @@ else REDIS_BACKEND="redis://${_APP_REDIS_USER}:${_APP_REDIS_PASS}@${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" fi -QUEUE='v1-database' APP_INCLUDE='/usr/src/code/app/workers/database.php' php /usr/src/code/vendor/bin/resque -dopcache.preload=opcache.preload=/usr/src/code/app/preload.php \ No newline at end of file +INTERVAL=0.1 QUEUE='v1-database' APP_INCLUDE='/usr/src/code/app/workers/database.php' php /usr/src/code/vendor/bin/resque -dopcache.preload=opcache.preload=/usr/src/code/app/preload.php \ No newline at end of file diff --git a/bin/worker-usage b/bin/worker-usage deleted file mode 100644 index 4174acce2..000000000 --- a/bin/worker-usage +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/sh - -if [ -z "$_APP_REDIS_USER" ] && [ -z "$_APP_REDIS_PASS" ] -then - REDIS_BACKEND="${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" -else - REDIS_BACKEND="redis://${_APP_REDIS_USER}:${_APP_REDIS_PASS}@${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" -fi - -INTERVAL=1 QUEUE='v1-usage' APP_INCLUDE='/usr/src/code/app/workers/usage.php' php /usr/src/code/vendor/bin/resque -dopcache.preload=opcache.preload=/usr/src/code/app/preload.php \ No newline at end of file diff --git a/composer.json b/composer.json index b3593e34d..76aa71cf7 100644 --- a/composer.json +++ b/composer.json @@ -45,7 +45,7 @@ "utopia-php/cache": "0.4.*", "utopia-php/cli": "0.11.*", "utopia-php/config": "0.2.*", - "utopia-php/database": "0.6.*", + "utopia-php/database": "0.7.*", "utopia-php/locale": "0.4.*", "utopia-php/registry": "0.5.*", "utopia-php/preloader": "0.2.*", @@ -53,6 +53,7 @@ "utopia-php/swoole": "0.2.*", "utopia-php/storage": "0.5.*", "utopia-php/image": "0.5.*", + "resque/php-resque": "1.3.6", "matomo/device-detector": "4.2.3", "dragonmantank/cron-expression": "3.1.0", diff --git a/composer.lock b/composer.lock index 9a8a5a4f1..db07f5df2 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "c2bda60e7b774a0c813f52033a268c6c", + "content-hash": "7de5dc8a9fe3cbc14696c685d1cdddee", "packages": [ { "name": "adhocore/jwt", @@ -1666,22 +1666,22 @@ }, { "name": "utopia-php/abuse", - "version": "0.6.1", + "version": "0.6.2", "source": { "type": "git", "url": "https://github.com/utopia-php/abuse.git", - "reference": "c9078aa3a87750d66060f0ed7642e03e5815da17" + "reference": "4cd9c16610f7398d2e1737663ef682fa721ae736" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/abuse/zipball/c9078aa3a87750d66060f0ed7642e03e5815da17", - "reference": "c9078aa3a87750d66060f0ed7642e03e5815da17", + "url": "https://api.github.com/repos/utopia-php/abuse/zipball/4cd9c16610f7398d2e1737663ef682fa721ae736", + "reference": "4cd9c16610f7398d2e1737663ef682fa721ae736", "shasum": "" }, "require": { "ext-pdo": "*", "php": ">=7.4", - "utopia-php/database": "0.6.*" + "utopia-php/database": "0.7.*" }, "require-dev": { "phpunit/phpunit": "^9.4", @@ -1713,9 +1713,9 @@ ], "support": { "issues": "https://github.com/utopia-php/abuse/issues", - "source": "https://github.com/utopia-php/abuse/tree/0.6.1" + "source": "https://github.com/utopia-php/abuse/tree/0.6.2" }, - "time": "2021-08-03T19:31:07+00:00" + "time": "2021-08-13T07:52:34+00:00" }, { "name": "utopia-php/analytics", @@ -1774,22 +1774,22 @@ }, { "name": "utopia-php/audit", - "version": "0.6.1", + "version": "0.6.2", "source": { "type": "git", "url": "https://github.com/utopia-php/audit.git", - "reference": "971dcd5c88309656df31ac20f326d3ac8b555594" + "reference": "2ec39a53eb98a5f9d230550ad56c7c04de5d77df" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/audit/zipball/971dcd5c88309656df31ac20f326d3ac8b555594", - "reference": "971dcd5c88309656df31ac20f326d3ac8b555594", + "url": "https://api.github.com/repos/utopia-php/audit/zipball/2ec39a53eb98a5f9d230550ad56c7c04de5d77df", + "reference": "2ec39a53eb98a5f9d230550ad56c7c04de5d77df", "shasum": "" }, "require": { "ext-pdo": "*", "php": ">=7.4", - "utopia-php/database": "0.6.*" + "utopia-php/database": "0.7.*" }, "require-dev": { "phpunit/phpunit": "^9.3", @@ -1821,9 +1821,9 @@ ], "support": { "issues": "https://github.com/utopia-php/audit/issues", - "source": "https://github.com/utopia-php/audit/tree/0.6.1" + "source": "https://github.com/utopia-php/audit/tree/0.6.2" }, - "time": "2021-08-03T19:29:34+00:00" + "time": "2021-08-13T08:05:20+00:00" }, { "name": "utopia-php/cache", @@ -1984,16 +1984,16 @@ }, { "name": "utopia-php/database", - "version": "0.6.1", + "version": "0.7.0", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "59d9d34164b6fb896bc43085a9a82a292b43473a" + "reference": "46c4a99347397e362a9429826e1888b0aefb2056" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/59d9d34164b6fb896bc43085a9a82a292b43473a", - "reference": "59d9d34164b6fb896bc43085a9a82a292b43473a", + "url": "https://api.github.com/repos/utopia-php/database/zipball/46c4a99347397e362a9429826e1888b0aefb2056", + "reference": "46c4a99347397e362a9429826e1888b0aefb2056", "shasum": "" }, "require": { @@ -2041,9 +2041,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/0.6.1" + "source": "https://github.com/utopia-php/database/tree/0.7.0" }, - "time": "2021-08-05T17:19:16+00:00" + "time": "2021-08-10T19:09:58+00:00" }, { "name": "utopia-php/domains", diff --git a/docker-compose.yml b/docker-compose.yml index 98f5f8f85..824543020 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -122,26 +122,6 @@ services: - _APP_FUNCTIONS_MEMORY - _APP_FUNCTIONS_MEMORY_SWAP - _APP_FUNCTIONS_RUNTIMES - - appwrite-worker-usage: - entrypoint: worker-usage - container_name: appwrite-worker-usage - build: - context: . - networks: - - appwrite - volumes: - - ./app:/usr/src/code/app - - ./src:/usr/src/code/src - depends_on: - - redis - - telegraf - environment: - - _APP_ENV - - _APP_REDIS_HOST - - _APP_REDIS_PORT - - _APP_REDIS_USER - - _APP_REDIS_PASS - _APP_STATSD_HOST - _APP_STATSD_PORT @@ -309,6 +289,8 @@ services: - _APP_FUNCTIONS_MEMORY - _APP_FUNCTIONS_MEMORY_SWAP - _APP_USAGE_STATS + - _APP_STATSD_HOST + - _APP_STATSD_PORT - DOCKERHUB_PULL_USERNAME - DOCKERHUB_PULL_PASSWORD diff --git a/public/dist/scripts/app-all.js b/public/dist/scripts/app-all.js index 569a6d46f..306e7c902 100644 --- a/public/dist/scripts/app-all.js +++ b/public/dist/scripts/app-all.js @@ -2375,8 +2375,10 @@ code.innerHTML=value;Prism.highlightElement(code);div.scrollTop=0;};element.addE function syncA(){element.value=picker.value;update();} function syncB(){picker.value=element.value;} element.parentNode.insertBefore(preview,element);update();syncB();}});})(window);(function(window){"use strict";window.ls.container.get("view").add({selector:"data-forms-copy",controller:function(element,alerts,document,window){var button=window.document.createElement("i");button.type="button";button.className="icon-docs note copy";button.style.cursor="pointer";element.parentNode.insertBefore(button,element.nextSibling);var copy=function(event){let disabled=element.disabled;element.disabled=false;element.focus();element.select();document.execCommand("Copy");if(document.selection){document.selection.empty();}else if(window.getSelection){window.getSelection().removeAllRanges();} -element.disabled=disabled;element.blur();alerts.add({text:"Copied to clipboard",class:""},3000);};button.addEventListener("click",copy);}});})(window);(function(window){"use strict";window.ls.container.get("view").add({selector:"data-custom-id",controller:function(element,sdk,console,window){let prevData="";let idType=element.getAttribute('data-id-type');const div=window.document.createElement("div");div.className="input-copy";const button=window.document.createElement("i");button.type="button";button.style.cursor="pointer";const writer=window.document.createElement("input");writer.type="text";writer.setAttribute("maxlength",element.getAttribute("maxlength"));const placeholder=element.getAttribute("placeholder");if(placeholder){writer.setAttribute("placeholder",placeholder);} -const info=window.document.createElement("div");info.className="text-fade text-size-xs margin-top-negative-small margin-bottom";div.appendChild(writer);div.appendChild(button);element.parentNode.insertBefore(div,element);element.parentNode.insertBefore(info,div.nextSibling);const switchType=function(event){if(idType=="custom"){idType="auto";setIdType(idType);}else{idType="custom";setIdType(idType);}} +element.disabled=disabled;element.blur();alerts.add({text:"Copied to clipboard",class:""},3000);};button.addEventListener("click",copy);}});})(window);(function(window){"use strict";window.ls.container.get("view").add({selector:"data-custom-id",controller:function(element,sdk,console,window){let prevData="";let idType=element.getAttribute('data-id-type');let disableSwitch=element.getAttribute('data-disable-switch');const div=window.document.createElement("div");if(disableSwitch!=="true"){div.className="input-copy";} +const button=window.document.createElement("i");button.type="button";button.style.cursor="pointer";const writer=window.document.createElement("input");writer.type="text";writer.setAttribute("maxlength",element.getAttribute("maxlength"));const placeholder=element.getAttribute("placeholder");if(placeholder){writer.setAttribute("placeholder",placeholder);} +const info=window.document.createElement("div");info.className="text-fade text-size-xs margin-top-negative-small margin-bottom";div.appendChild(writer);if(disableSwitch!=="true"){div.appendChild(button);} +element.parentNode.insertBefore(div,element);element.parentNode.insertBefore(info,div.nextSibling);const switchType=function(event){if(idType=="custom"){idType="auto";setIdType(idType);}else{idType="custom";setIdType(idType);}} const validate=function(event){const[service,method]=element.dataset["validator"].split('.');const value=event.target.value;if(value.length<1){event.target.setCustomValidity("ID is required");}else{switch(service){case'projects':setValidity(console[service][method](value),event.target);break;default:setValidity(sdk[service][method](value),event.target);}}} const setValidity=async function(promise,target){try{await promise;target.setCustomValidity("ID already exists");}catch(e){target.setCustomValidity("");}} const setIdType=function(idType){if(idType=="custom"){element.setAttribute("data-id-type",idType);info.innerHTML="Allowed Characters A-Z, a-z, 0-9, and non-leading underscore";if(prevData==='auto-generated'){prevData=""} diff --git a/public/dist/scripts/app.js b/public/dist/scripts/app.js index 4f987bce4..a41810162 100644 --- a/public/dist/scripts/app.js +++ b/public/dist/scripts/app.js @@ -358,8 +358,10 @@ code.innerHTML=value;Prism.highlightElement(code);div.scrollTop=0;};element.addE function syncA(){element.value=picker.value;update();} function syncB(){picker.value=element.value;} element.parentNode.insertBefore(preview,element);update();syncB();}});})(window);(function(window){"use strict";window.ls.container.get("view").add({selector:"data-forms-copy",controller:function(element,alerts,document,window){var button=window.document.createElement("i");button.type="button";button.className="icon-docs note copy";button.style.cursor="pointer";element.parentNode.insertBefore(button,element.nextSibling);var copy=function(event){let disabled=element.disabled;element.disabled=false;element.focus();element.select();document.execCommand("Copy");if(document.selection){document.selection.empty();}else if(window.getSelection){window.getSelection().removeAllRanges();} -element.disabled=disabled;element.blur();alerts.add({text:"Copied to clipboard",class:""},3000);};button.addEventListener("click",copy);}});})(window);(function(window){"use strict";window.ls.container.get("view").add({selector:"data-custom-id",controller:function(element,sdk,console,window){let prevData="";let idType=element.getAttribute('data-id-type');const div=window.document.createElement("div");div.className="input-copy";const button=window.document.createElement("i");button.type="button";button.style.cursor="pointer";const writer=window.document.createElement("input");writer.type="text";writer.setAttribute("maxlength",element.getAttribute("maxlength"));const placeholder=element.getAttribute("placeholder");if(placeholder){writer.setAttribute("placeholder",placeholder);} -const info=window.document.createElement("div");info.className="text-fade text-size-xs margin-top-negative-small margin-bottom";div.appendChild(writer);div.appendChild(button);element.parentNode.insertBefore(div,element);element.parentNode.insertBefore(info,div.nextSibling);const switchType=function(event){if(idType=="custom"){idType="auto";setIdType(idType);}else{idType="custom";setIdType(idType);}} +element.disabled=disabled;element.blur();alerts.add({text:"Copied to clipboard",class:""},3000);};button.addEventListener("click",copy);}});})(window);(function(window){"use strict";window.ls.container.get("view").add({selector:"data-custom-id",controller:function(element,sdk,console,window){let prevData="";let idType=element.getAttribute('data-id-type');let disableSwitch=element.getAttribute('data-disable-switch');const div=window.document.createElement("div");if(disableSwitch!=="true"){div.className="input-copy";} +const button=window.document.createElement("i");button.type="button";button.style.cursor="pointer";const writer=window.document.createElement("input");writer.type="text";writer.setAttribute("maxlength",element.getAttribute("maxlength"));const placeholder=element.getAttribute("placeholder");if(placeholder){writer.setAttribute("placeholder",placeholder);} +const info=window.document.createElement("div");info.className="text-fade text-size-xs margin-top-negative-small margin-bottom";div.appendChild(writer);if(disableSwitch!=="true"){div.appendChild(button);} +element.parentNode.insertBefore(div,element);element.parentNode.insertBefore(info,div.nextSibling);const switchType=function(event){if(idType=="custom"){idType="auto";setIdType(idType);}else{idType="custom";setIdType(idType);}} const validate=function(event){const[service,method]=element.dataset["validator"].split('.');const value=event.target.value;if(value.length<1){event.target.setCustomValidity("ID is required");}else{switch(service){case'projects':setValidity(console[service][method](value),event.target);break;default:setValidity(sdk[service][method](value),event.target);}}} const setValidity=async function(promise,target){try{await promise;target.setCustomValidity("ID already exists");}catch(e){target.setCustomValidity("");}} const setIdType=function(idType){if(idType=="custom"){element.setAttribute("data-id-type",idType);info.innerHTML="Allowed Characters A-Z, a-z, 0-9, and non-leading underscore";if(prevData==='auto-generated'){prevData=""} diff --git a/public/scripts/views/forms/custom-id.js b/public/scripts/views/forms/custom-id.js index c444912c2..304f78000 100644 --- a/public/scripts/views/forms/custom-id.js +++ b/public/scripts/views/forms/custom-id.js @@ -5,9 +5,12 @@ controller: function (element, sdk, console, window) { let prevData = ""; let idType = element.getAttribute('data-id-type'); + let disableSwitch = element.getAttribute('data-disable-switch'); const div = window.document.createElement("div"); - div.className = "input-copy"; + if(disableSwitch !== "true") { + div.className = "input-copy"; + } const button = window.document.createElement("i"); button.type = "button"; @@ -25,7 +28,9 @@ info.className = "text-fade text-size-xs margin-top-negative-small margin-bottom"; div.appendChild(writer); - div.appendChild(button); + if(disableSwitch !== "true") { + div.appendChild(button); + } element.parentNode.insertBefore(div, element); element.parentNode.insertBefore(info, div.nextSibling); diff --git a/src/Appwrite/Stats/Stats.php b/src/Appwrite/Stats/Stats.php new file mode 100644 index 000000000..b07c4c85c --- /dev/null +++ b/src/Appwrite/Stats/Stats.php @@ -0,0 +1,129 @@ +statsd = $statsd; + } + + /** + * @param string $key + * @param mixed $value + * + * @return $this + */ + public function setParam(string $key, $value): self + { + $this->params[$key] = $value; + + return $this; + } + + /** + * @param string $key + * + * @return mixed|null + */ + public function getParam(string $key) + { + return (isset($this->params[$key])) ? $this->params[$key] : null; + } + + /** + * @param string $namespace + * + * @return $this + */ + public function setNamespace(string $namespace): self + { + $this->namespace = $namespace; + + return $this; + } + + /** + * @return string + */ + public function getNamespace() + { + return $this->namespace; + } + + /** + * Submit data to StatsD. + */ + public function submit(): void + { + $projectId = $this->params['projectId'] ?? ''; + + $storage = $this->params['storage'] ?? 0; + + $networkRequestSize = $this->params['networkRequestSize'] ?? 0; + $networkResponseSize = $this->params['networkResponseSize'] ?? 0; + + $httpMethod = $this->params['httpMethod'] ?? ''; + $httpRequest = $this->params['httpRequest'] ?? 0; + + $functionId = $this->params['functionId'] ?? ''; + $functionExecution = $this->params['functionExecution'] ?? 0; + $functionExecutionTime = $this->params['functionExecutionTime'] ?? 0; + $functionStatus = $this->params['functionStatus'] ?? ''; + + $tags = ",project={$projectId},version=" . App::getEnv('_APP_VERSION', 'UNKNOWN'); + + // the global namespace is prepended to every key (optional) + $this->statsd->setNamespace($this->namespace); + + if ($httpRequest >= 1) { + $this->statsd->increment('requests.all' . $tags . ',method=' . \strtolower($httpMethod)); + } + + if ($functionExecution >= 1) { + $this->statsd->increment('executions.all' . $tags . ',functionId=' . $functionId . ',functionStatus=' . $functionStatus); + $this->statsd->count('executions.time' . $tags . ',functionId=' . $functionId, $functionExecutionTime); + } + + $this->statsd->count('network.inbound' . $tags, $networkRequestSize); + $this->statsd->count('network.outbound' . $tags, $networkResponseSize); + $this->statsd->count('network.all' . $tags, $networkRequestSize + $networkResponseSize); + + if ($storage >= 1) { + $this->statsd->count('storage.all' . $tags, $storage); + } + + $this->reset(); + } + + public function reset(): self + { + $this->params = []; + $this->namespace = 'appwrite.usage'; + + return $this; + } +} diff --git a/tests/e2e/Services/Database/DatabaseBase.php b/tests/e2e/Services/Database/DatabaseBase.php index ab6b08120..74d5af583 100644 --- a/tests/e2e/Services/Database/DatabaseBase.php +++ b/tests/e2e/Services/Database/DatabaseBase.php @@ -43,16 +43,19 @@ trait DatabaseBase 'required' => true, ]); + sleep(2); + $releaseYear = $this->client->call(Client::METHOD_POST, '/database/collections/' . $data['moviesId'] . '/attributes/integer', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], 'x-appwrite-key' => $this->getProject()['apiKey'] ]), [ 'attributeId' => 'releaseYear', - 'size' => 0, 'required' => true, ]); + sleep(2); + $actors = $this->client->call(Client::METHOD_POST, '/database/collections/' . $data['moviesId'] . '/attributes/string', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], @@ -61,7 +64,6 @@ trait DatabaseBase 'attributeId' => 'actors', 'size' => 256, 'required' => false, - 'default' => null, 'array' => true, ]); @@ -88,7 +90,7 @@ trait DatabaseBase $this->assertEquals($actors['body']['array'], true); // wait for database worker to create attributes - sleep(10); + sleep(5); $movies = $this->client->call(Client::METHOD_GET, '/database/collections/' . $data['moviesId'], array_merge([ 'content-type' => 'application/json', @@ -120,7 +122,7 @@ trait DatabaseBase 'x-appwrite-project' => $this->getProject()['$id'], 'x-appwrite-key' => $this->getProject()['apiKey'] ]), [ - 'id' => 'titleIndex', + 'indexId' => 'titleIndex', 'type' => 'fulltext', 'attributes' => ['title'], ]); @@ -645,7 +647,7 @@ trait DatabaseBase // $this->assertEquals('Minimum value must be lesser than maximum value', $invalidRange['body']['message']); // wait for worker to add attributes - sleep(10); + sleep(15); $collection = $this->client->call(Client::METHOD_GET, '/database/collections/' . $collectionId, array_merge([ 'content-type' => 'application/json', @@ -654,7 +656,7 @@ trait DatabaseBase ]), []); $this->assertCount(7, $collection['body']['attributes']); - $this->assertCount(0, $collection['body']['attributesInQueue']); + // $this->assertCount(0, $collection['body']['attributesInQueue']); /** * Test for successful validation diff --git a/tests/e2e/Services/Database/DatabaseCustomServerTest.php b/tests/e2e/Services/Database/DatabaseCustomServerTest.php index dea3b588c..421055fde 100644 --- a/tests/e2e/Services/Database/DatabaseCustomServerTest.php +++ b/tests/e2e/Services/Database/DatabaseCustomServerTest.php @@ -13,7 +13,7 @@ class DatabaseCustomServerTest extends Scope use ProjectCustom; use SideServer; - public function testDeleteCollection() + public function testDeleteAttribute(): array { /** * Test for SUCCESS @@ -54,7 +54,59 @@ class DatabaseCustomServerTest extends Scope 'required' => true, ]); - // wait for database worker to finish creating attributes + $unneeded = $this->client->call(Client::METHOD_POST, '/database/collections/' . $actors['body']['$id'] . '/attributes/string', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'attributeId' => 'unneeded', + 'size' => 256, + 'required' => true, + ]); + + // Wait for database worker to finish creating attributes + sleep(5); + + $index = $this->client->call(Client::METHOD_POST, '/database/collections/' . $actors['body']['$id'] . '/indexes', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'indexId' => 'key_lastName', + 'type' => 'key', + 'attributes' => [ + 'lastName', + ], + ]); + + // Wait for database worker to finish creating index + sleep(5); + + $collection = $this->client->call(Client::METHOD_GET, '/database/collections/' . $actors['body']['$id'], array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), []); + + $unneededId = $unneeded['body']['$id']; + + $this->assertEquals($collection['body']['$id'], $firstName['body']['$collection']); + $this->assertEquals($collection['body']['$id'], $lastName['body']['$collection']); + $this->assertIsArray($collection['body']['attributes']); + $this->assertCount(3, $collection['body']['attributes']); + $this->assertEquals($collection['body']['attributes'][0]['$id'], $firstName['body']['$id']); + $this->assertEquals($collection['body']['attributes'][1]['$id'], $lastName['body']['$id']); + $this->assertEquals($collection['body']['attributes'][2]['$id'], $unneeded['body']['$id']); + $this->assertCount(1, $collection['body']['indexes']); + $this->assertEquals($collection['body']['indexes'][0]['$id'], $index['body']['$id']); + + // Delete attribute + $this->client->call(Client::METHOD_DELETE, '/database/collections/' . $actors ['body']['$id'] . '/attributes/' . $unneededId, array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ])); + sleep(5); $collection = $this->client->call(Client::METHOD_GET, '/database/collections/' . $actors['body']['$id'], array_merge([ @@ -70,8 +122,45 @@ class DatabaseCustomServerTest extends Scope $this->assertEquals($collection['body']['attributes'][0]['$id'], $firstName['body']['$id']); $this->assertEquals($collection['body']['attributes'][1]['$id'], $lastName['body']['$id']); + return [ + 'collectionId' => $actors['body']['$id'], + 'indexId' => $index['body']['$id'], + ]; + } + /** + * @depends testDeleteAttribute + */ + public function testDeleteIndex($data): array + { + $index = $this->client->call(Client::METHOD_DELETE, '/database/collections/' . $data['collectionId'] . '/indexes/'. $data['indexId'], array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ])); + + // Wait for database worker to finish deleting index + sleep(5); + + $collection = $this->client->call(Client::METHOD_GET, '/database/collections/' . $data['collectionId'], array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), []); + + $this->assertCount(0, $collection['body']['indexes']); + + return $data; + } + + /** + * @depends testDeleteIndex + */ + public function testDeleteCollection($data) + { + $collectionId = $data['collectionId']; + // Add Documents to the collection - $document1 = $this->client->call(Client::METHOD_POST, '/database/collections/' . $actors['body']['$id'] . '/documents', array_merge([ + $document1 = $this->client->call(Client::METHOD_POST, '/database/collections/' . $collectionId . '/documents', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), [ @@ -84,7 +173,7 @@ class DatabaseCustomServerTest extends Scope 'write' => ['user:'.$this->getUser()['$id']], ]); - $document2 = $this->client->call(Client::METHOD_POST, '/database/collections/' . $actors['body']['$id'] . '/documents', array_merge([ + $document2 = $this->client->call(Client::METHOD_POST, '/database/collections/' . $collectionId . '/documents', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), [ @@ -98,7 +187,7 @@ class DatabaseCustomServerTest extends Scope ]); $this->assertEquals($document1['headers']['status-code'], 201); - $this->assertEquals($document1['body']['$collection'], $actors['body']['$id']); + $this->assertEquals($document1['body']['$collection'], $collectionId); $this->assertIsArray($document1['body']['$read']); $this->assertIsArray($document1['body']['$write']); $this->assertCount(1, $document1['body']['$read']); @@ -107,7 +196,7 @@ class DatabaseCustomServerTest extends Scope $this->assertEquals($document1['body']['lastName'], 'Holland'); $this->assertEquals($document2['headers']['status-code'], 201); - $this->assertEquals($document2['body']['$collection'], $actors['body']['$id']); + $this->assertEquals($document2['body']['$collection'], $collectionId); $this->assertIsArray($document2['body']['$read']); $this->assertIsArray($document2['body']['$write']); $this->assertCount(1, $document2['body']['$read']); @@ -116,7 +205,7 @@ class DatabaseCustomServerTest extends Scope $this->assertEquals($document2['body']['lastName'], 'Jackson'); // Delete the actors collection - $response = $this->client->call(Client::METHOD_DELETE, '/database/collections/'.$actors['body']['$id'], array_merge([ + $response = $this->client->call(Client::METHOD_DELETE, '/database/collections/' . $collectionId , array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], 'x-appwrite-key' => $this->getProject()['apiKey'] @@ -126,7 +215,7 @@ class DatabaseCustomServerTest extends Scope $this->assertEquals($response['body'],""); // Try to get the collection and check if it has been deleted - $response = $this->client->call(Client::METHOD_GET, '/database/collections/'.$actors['body']['$id'], array_merge([ + $response = $this->client->call(Client::METHOD_GET, '/database/collections/' . $collectionId , array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'] ], $this->getHeaders())); diff --git a/tests/e2e/Services/Webhooks/WebhooksCustomServerTest.php b/tests/e2e/Services/Webhooks/WebhooksCustomServerTest.php index 923039fa1..bcb473b4a 100644 --- a/tests/e2e/Services/Webhooks/WebhooksCustomServerTest.php +++ b/tests/e2e/Services/Webhooks/WebhooksCustomServerTest.php @@ -63,7 +63,7 @@ class WebhooksCustomServerTest extends Scope 'x-appwrite-project' => $this->getProject()['$id'], 'x-appwrite-key' => $this->getProject()['apiKey'] ]), [ - 'id' => 'fullname', + 'indexId' => 'fullname', 'type' => 'key', 'attributes' => ['lastName', 'firstName'], 'orders' => ['ASC', 'ASC'], diff --git a/tests/unit/Stats/StatsTest.php b/tests/unit/Stats/StatsTest.php new file mode 100644 index 000000000..784ac77fb --- /dev/null +++ b/tests/unit/Stats/StatsTest.php @@ -0,0 +1,69 @@ +object = new Stats($statsd); + } + + public function tearDown(): void + { + } + + public function testNamespace() + { + $this->object->setNamespace('appwritetest.usage'); + $this->assertEquals('appwritetest.usage', $this->object->getNamespace()); + } + + public function testParams() + { + $this->object + ->setParam('projectId', 'appwrite_test') + ->setParam('networkRequestSize', 100) + ; + + $this->assertEquals('appwrite_test', $this->object->getParam('projectId')); + $this->assertEquals(100, $this->object->getParam('networkRequestSize')); + + $this->object->submit(); + + $this->assertEquals(null, $this->object->getParam('projectId')); + $this->assertEquals(null, $this->object->getParam('networkRequestSize')); + } + + public function testReset() + { + $this->object + ->setParam('projectId', 'appwrite_test') + ->setParam('networkRequestSize', 100) + ; + + $this->assertEquals('appwrite_test', $this->object->getParam('projectId')); + $this->assertEquals(100, $this->object->getParam('networkRequestSize')); + + $this->object->reset(); + + $this->assertEquals(null, $this->object->getParam('projectId')); + $this->assertEquals(null, $this->object->getParam('networkRequestSize')); + $this->assertEquals('appwrite.usage', $this->object->getNamespace()); + } +}