diff --git a/.env b/.env index 228b7f39d4..3e9a4c53a0 100644 --- a/.env +++ b/.env @@ -41,6 +41,9 @@ _APP_SMTP_PORT=1025 _APP_SMTP_SECURE= _APP_SMTP_USERNAME= _APP_SMTP_PASSWORD= +_APP_HAMSTER_INTERVAL=86400 +_APP_HAMSTER_TIME=21:00 +_APP_MIXPANEL_TOKEN= _APP_SMS_PROVIDER=sms://username:password@mock _APP_SMS_FROM=+123456789 _APP_STORAGE_LIMIT=30000000 diff --git a/.gitmodules b/.gitmodules index 0a7d2f297e..6785e119cb 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,4 +1,4 @@ [submodule "app/console"] path = app/console url = https://github.com/appwrite/console - branch = feat-cloud \ No newline at end of file + branch = 2.2.2 diff --git a/Dockerfile b/Dockerfile index 12a7ae5c19..5265a690f8 100755 --- a/Dockerfile +++ b/Dockerfile @@ -111,7 +111,7 @@ RUN mkdir -p /storage/uploads && \ # Executables RUN chmod +x /usr/local/bin/doctor && \ - chmod +x /usr/local/bin/patch-create-missing-schedules && \ + chmod +x /usr/local/bin/patch-delete-schedule-updated-at-attribute && \ chmod +x /usr/local/bin/maintenance && \ chmod +x /usr/local/bin/volume-sync && \ chmod +x /usr/local/bin/install && \ @@ -121,6 +121,7 @@ RUN chmod +x /usr/local/bin/doctor && \ chmod +x /usr/local/bin/sdks && \ chmod +x /usr/local/bin/specs && \ chmod +x /usr/local/bin/ssl && \ + chmod +x /usr/local/bin/hamster && \ chmod +x /usr/local/bin/test && \ chmod +x /usr/local/bin/vars && \ chmod +x /usr/local/bin/worker-audits && \ diff --git a/app/config/collections.php b/app/config/collections.php index e55c694ea3..5648bfe6ea 100644 --- a/app/config/collections.php +++ b/app/config/collections.php @@ -2311,17 +2311,6 @@ $collections = [ 'array' => false, 'filters' => [], ], - [ - '$id' => ID::custom('scheduleUpdatedAt'), - 'type' => Database::VAR_DATETIME, - 'format' => '', - 'size' => 0, - 'signed' => false, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => ['datetime'], - ], [ '$id' => ID::custom('timeout'), 'type' => Database::VAR_INTEGER, @@ -2616,6 +2605,17 @@ $collections = [ 'array' => false, 'filters' => ['datetime'], ], + [ + '$id' => ID::custom('endTime'), + 'type' => Database::VAR_DATETIME, + 'format' => '', + 'size' => 0, + 'signed' => false, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => ['datetime'], + ], [ '$id' => ID::custom('duration'), 'type' => Database::VAR_INTEGER, @@ -2672,7 +2672,7 @@ $collections = [ 'filters' => [], ], [ - '$id' => ID::custom('path'), + '$id' => ID::custom('outputPath'), 'type' => Database::VAR_STRING, 'format' => '', 'size' => 2048, @@ -2682,17 +2682,6 @@ $collections = [ 'array' => false, 'filters' => [], ], - [ - '$id' => ID::custom('size'), - 'type' => Database::VAR_INTEGER, - 'format' => '', - 'size' => 0, - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ], [ '$id' => ID::custom('stderr'), 'type' => Database::VAR_STRING, diff --git a/app/config/errors.php b/app/config/errors.php index a9aaa13ab3..4e8d341161 100644 --- a/app/config/errors.php +++ b/app/config/errors.php @@ -88,6 +88,11 @@ return [ 'description' => 'The request cannot be fulfilled with the current protocol. Please check the value of the _APP_OPTIONS_FORCE_HTTPS environment variable.', 'code' => 500, ], + Exception::GENERAL_CODES_DISABLED => [ + 'name' => Exception::GENERAL_CODES_DISABLED, + 'description' => 'Invitation codes are disabled on this server. Please contact the server administrator.', + 'code' => 500, + ], /** User Errors */ Exception::USER_COUNT_EXCEEDED => [ @@ -125,8 +130,8 @@ return [ 'description' => 'Console registration is restricted to specific emails. Contact your administrator for more information.', 'code' => 401, ], - Exception::USER_CODE_INVALID => [ - 'name' => Exception::USER_CODE_INVALID, + Exception::USER_INVALID_CODE => [ + 'name' => Exception::USER_INVALID_CODE, 'description' => 'The specified code is not valid. Contact your administrator for more information.', 'code' => 401, ], diff --git a/app/console b/app/console index fae048b917..39727607a0 160000 --- a/app/console +++ b/app/console @@ -1 +1 @@ -Subproject commit fae048b91787d0b372c13caf27f14fc8b780ef60 +Subproject commit 39727607a036534a323101fea876cbe68b05f4f0 diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index 8862f63c1f..465c6f335a 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -81,8 +81,12 @@ App::post('/v1/account/invite') $whitelistCodes = (!empty(App::getEnv('_APP_CONSOLE_WHITELIST_CODES', null))) ? \explode(',', App::getEnv('_APP_CONSOLE_WHITELIST_CODES', null)) : []; + if (empty($whitelistCodes)) { + throw new Exception(Exception::GENERAL_CODES_DISABLED); + } + if (!empty($whitelistCodes) && !\in_array($code, $whitelistCodes)) { - throw new Exception(Exception::USER_CODE_INVALID); + throw new Exception(Exception::USER_INVALID_CODE); } $limit = $project->getAttribute('auths', [])['limit'] ?? 0; @@ -168,7 +172,7 @@ App::post('/v1/account') $whitelistEmails = $project->getAttribute('authWhitelistEmails'); $whitelistIPs = $project->getAttribute('authWhitelistIPs'); - if (!empty($whitelistEmails) && !\in_array($email, $whitelistEmails)) { + if (!empty($whitelistEmails) && !\in_array($email, $whitelistEmails) && !\in_array(strtoupper($email), $whitelistEmails)) { throw new Exception(Exception::USER_EMAIL_NOT_WHITELISTED); } diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 1ae5b39204..a2971ea4c9 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -86,7 +86,6 @@ App::post('/v1/functions') 'events' => $events, 'schedule' => $schedule, 'scheduleInternalId' => '', - 'scheduleUpdatedAt' => DateTime::now(), 'timeout' => $timeout, 'search' => implode(' ', [$functionId, $name, $runtime]) ])); @@ -416,26 +415,16 @@ App::put('/v1/functions/:functionId') 'name' => $name, 'events' => $events, 'schedule' => $schedule, - 'scheduleUpdatedAt' => DateTime::now(), 'timeout' => $timeout, 'enabled' => $enabled, 'search' => implode(' ', [$functionId, $name, $function->getAttribute('runtime')]), ]))); $schedule = $dbForConsole->getDocument('schedules', $function->getAttribute('scheduleId')); - - /** - * In case we want to clear the schedule - */ - if (!empty($function->getAttribute('deployment'))) { - $schedule->setAttribute('resourceUpdatedAt', $function->getAttribute('scheduleUpdatedAt')); - } - $schedule + ->setAttribute('resourceUpdatedAt', DateTime::now()) ->setAttribute('schedule', $function->getAttribute('schedule')) ->setAttribute('active', !empty($function->getAttribute('schedule')) && !empty($function->getAttribute('deployment'))); - - Authorization::skip(fn () => $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule)); $eventsInstance->setParam('functionId', $function->getId()); @@ -488,19 +477,14 @@ App::patch('/v1/functions/:functionId/deployments/:deploymentId') $function = $dbForProject->updateDocument('functions', $function->getId(), new Document(array_merge($function->getArrayCopy(), [ 'deploymentInternalId' => $deployment->getInternalId(), - 'deployment' => $deployment->getId() + 'deployment' => $deployment->getId(), ]))); $schedule = $dbForConsole->getDocument('schedules', $function->getAttribute('scheduleId')); - - $active = !empty($function->getAttribute('schedule')); - - if ($active) { - $schedule->setAttribute('resourceUpdatedAt', datetime::now()); - } - - $schedule->setAttribute('active', $active); - + $schedule + ->setAttribute('resourceUpdatedAt', DateTime::now()) + ->setAttribute('schedule', $function->getAttribute('schedule')) + ->setAttribute('active', !empty($function->getAttribute('schedule')) && !empty($function->getAttribute('deployment'))); Authorization::skip(fn () => $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule)); $events @@ -543,12 +527,9 @@ App::delete('/v1/functions/:functionId') } $schedule = $dbForConsole->getDocument('schedules', $function->getAttribute('scheduleId')); - $schedule ->setAttribute('resourceUpdatedAt', DateTime::now()) - ->setAttribute('active', false) - ; - + ->setAttribute('active', false); Authorization::skip(fn () => $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule)); $deletes @@ -749,22 +730,6 @@ App::post('/v1/functions/:functionId/deployments') } } - /** - * TODO Should we update also the function collection with the scheduleUpdatedAt attr? - */ - - $schedule = $dbForConsole->getDocument('schedules', $function->getAttribute('scheduleId')); - - $active = !empty($function->getAttribute('schedule')); - - if ($active) { - $schedule->setAttribute('resourceUpdatedAt', datetime::now()); - } - - $schedule->setAttribute('active', $active); - - Authorization::skip(fn () => $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule)); - $metadata = null; $events @@ -1163,7 +1128,7 @@ App::post('/v1/functions/:functionId/executions') variables: $vars, timeout: $function->getAttribute('timeout', 0), image: $runtime['image'], - source: $build->getAttribute('path', ''), + source: $build->getAttribute('outputPath', ''), entrypoint: $deployment->getAttribute('entrypoint', ''), ); @@ -1347,7 +1312,8 @@ App::post('/v1/functions/:functionId/variables') ->param('value', null, new Text(8192), 'Variable value. Max length: 8192 chars.', false) ->inject('response') ->inject('dbForProject') - ->action(function (string $functionId, string $key, string $value, Response $response, Database $dbForProject) { + ->inject('dbForConsole') + ->action(function (string $functionId, string $key, string $value, Response $response, Database $dbForProject, Database $dbForConsole) { $function = $dbForProject->getDocument('functions', $functionId); if ($function->isEmpty()) { @@ -1376,8 +1342,22 @@ App::post('/v1/functions/:functionId/variables') throw new Exception(Exception::VARIABLE_ALREADY_EXISTS); } + $schedule = $dbForConsole->getDocument('schedules', $function->getAttribute('scheduleId')); + $schedule + ->setAttribute('resourceUpdatedAt', DateTime::now()) + ->setAttribute('schedule', $function->getAttribute('schedule')) + ->setAttribute('active', !empty($function->getAttribute('schedule')) && !empty($function->getAttribute('deployment'))); + Authorization::skip(fn () => $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule)); + $dbForProject->deleteCachedDocument('functions', $function->getId()); + $schedule = $dbForConsole->getDocument('schedules', $function->getAttribute('scheduleId')); + $schedule + ->setAttribute('resourceUpdatedAt', DateTime::now()) + ->setAttribute('schedule', $function->getAttribute('schedule')) + ->setAttribute('active', !empty($function->getAttribute('schedule')) && !empty($function->getAttribute('deployment'))); + Authorization::skip(fn () => $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule)); + $response ->setStatusCode(Response::STATUS_CODE_CREATED) ->dynamic($variable, Response::MODEL_VARIABLE); @@ -1463,7 +1443,8 @@ App::put('/v1/functions/:functionId/variables/:variableId') ->param('value', null, new Text(8192), 'Variable value. Max length: 8192 chars.', true) ->inject('response') ->inject('dbForProject') - ->action(function (string $functionId, string $variableId, string $key, ?string $value, Response $response, Database $dbForProject) { + ->inject('dbForConsole') + ->action(function (string $functionId, string $variableId, string $key, ?string $value, Response $response, Database $dbForProject, Database $dbForConsole) { $function = $dbForProject->getDocument('functions', $functionId); @@ -1492,8 +1473,22 @@ App::put('/v1/functions/:functionId/variables/:variableId') throw new Exception(Exception::VARIABLE_ALREADY_EXISTS); } + $schedule = $dbForConsole->getDocument('schedules', $function->getAttribute('scheduleId')); + $schedule + ->setAttribute('resourceUpdatedAt', DateTime::now()) + ->setAttribute('schedule', $function->getAttribute('schedule')) + ->setAttribute('active', !empty($function->getAttribute('schedule')) && !empty($function->getAttribute('deployment'))); + Authorization::skip(fn () => $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule)); + $dbForProject->deleteCachedDocument('functions', $function->getId()); + $schedule = $dbForConsole->getDocument('schedules', $function->getAttribute('scheduleId')); + $schedule + ->setAttribute('resourceUpdatedAt', DateTime::now()) + ->setAttribute('schedule', $function->getAttribute('schedule')) + ->setAttribute('active', !empty($function->getAttribute('schedule')) && !empty($function->getAttribute('deployment'))); + Authorization::skip(fn () => $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule)); + $response->dynamic($variable, Response::MODEL_VARIABLE); }); @@ -1513,7 +1508,8 @@ App::delete('/v1/functions/:functionId/variables/:variableId') ->param('variableId', '', new UID(), 'Variable unique ID.', false) ->inject('response') ->inject('dbForProject') - ->action(function (string $functionId, string $variableId, Response $response, Database $dbForProject) { + ->inject('dbForConsole') + ->action(function (string $functionId, string $variableId, Response $response, Database $dbForProject, Database $dbForConsole) { $function = $dbForProject->getDocument('functions', $functionId); if ($function->isEmpty()) { @@ -1530,7 +1526,22 @@ App::delete('/v1/functions/:functionId/variables/:variableId') } $dbForProject->deleteDocument('variables', $variable->getId()); + + $schedule = $dbForConsole->getDocument('schedules', $function->getAttribute('scheduleId')); + $schedule + ->setAttribute('resourceUpdatedAt', DateTime::now()) + ->setAttribute('schedule', $function->getAttribute('schedule')) + ->setAttribute('active', !empty($function->getAttribute('schedule')) && !empty($function->getAttribute('deployment'))); + Authorization::skip(fn () => $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule)); + $dbForProject->deleteCachedDocument('functions', $function->getId()); + $schedule = $dbForConsole->getDocument('schedules', $function->getAttribute('scheduleId')); + $schedule + ->setAttribute('resourceUpdatedAt', DateTime::now()) + ->setAttribute('schedule', $function->getAttribute('schedule')) + ->setAttribute('active', !empty($function->getAttribute('schedule')) && !empty($function->getAttribute('deployment'))); + Authorization::skip(fn () => $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule)); + $response->noContent(); }); diff --git a/app/workers/builds.php b/app/workers/builds.php index ced0a84113..972eb5dd18 100644 --- a/app/workers/builds.php +++ b/app/workers/builds.php @@ -104,13 +104,13 @@ class BuildsV1 extends Worker 'deploymentInternalId' => $deployment->getInternalId(), 'deploymentId' => $deployment->getId(), 'status' => 'processing', - 'path' => '', - 'size' => 0, + 'outputPath' => '', 'runtime' => $function->getAttribute('runtime'), 'source' => $deployment->getAttribute('path'), 'sourceType' => $device, 'stdout' => '', 'stderr' => '', + 'endTime' => null, 'duration' => 0 ])); $deployment->setAttribute('buildId', $build->getId()); @@ -192,12 +192,14 @@ class BuildsV1 extends Worker ] ); + $endTime = new \DateTime(); + $endTime->setTimestamp($response['endTimeUnix']); + /** Update the build document */ - $build->setAttribute('startTime', DateTime::format((new \DateTime())->setTimestamp($response['startTime']))); + $build->setAttribute('endTime', DateTime::format($endTime)); $build->setAttribute('duration', \intval($response['duration'])); $build->setAttribute('status', $response['status']); - $build->setAttribute('path', $response['path']); - $build->setAttribute('size', $response['size']); + $build->setAttribute('outputPath', $response['outputPath']); $build->setAttribute('stderr', $response['stderr']); $build->setAttribute('stdout', $response['stdout']); @@ -206,8 +208,6 @@ class BuildsV1 extends Worker Console::success("Build id: $buildId created"); - $function->setAttribute('scheduleUpdatedAt', DateTime::now()); - /** Set auto deploy */ if ($deployment->getAttribute('activate') === true) { $function->setAttribute('deploymentInternalId', $deployment->getInternalId()); @@ -218,7 +218,7 @@ class BuildsV1 extends Worker /** Update function schedule */ $dbForConsole = $this->getConsoleDB(); $schedule = $dbForConsole->getDocument('schedules', $function->getAttribute('scheduleId')); - $schedule->setAttribute('resourceUpdatedAt', $function->getAttribute('scheduleUpdatedAt')); + $schedule->setAttribute('resourceUpdatedAt', DateTime::now()); $schedule ->setAttribute('schedule', $function->getAttribute('schedule')) @@ -229,7 +229,7 @@ class BuildsV1 extends Worker } catch (\Throwable $th) { $endTime = DateTime::now(); $interval = (new \DateTime($endTime))->diff(new \DateTime($startTime)); - + $build->setAttribute('endTime', $endTime); $build->setAttribute('duration', $interval->format('%s') + 0); $build->setAttribute('status', 'failed'); $build->setAttribute('stderr', $th->getMessage()); diff --git a/app/workers/deletes.php b/app/workers/deletes.php index f388389605..1f338087d9 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -527,10 +527,10 @@ class DeletesV1 extends Worker $this->deleteByGroup('builds', [ Query::equal('deploymentId', [$deploymentId]) ], $dbForProject, function (Document $document) use ($storageBuilds, $deploymentId) { - if ($storageBuilds->delete($document->getAttribute('path', ''), true)) { - Console::success('Deleted build files: ' . $document->getAttribute('path', '')); + if ($storageBuilds->delete($document->getAttribute('outputPath', ''), true)) { + Console::success('Deleted build files: ' . $document->getAttribute('outputPath', '')); } else { - Console::error('Failed to delete build files: ' . $document->getAttribute('path', '')); + Console::error('Failed to delete build files: ' . $document->getAttribute('outputPath', '')); } }); } @@ -576,10 +576,10 @@ class DeletesV1 extends Worker $this->deleteByGroup('builds', [ Query::equal('deploymentId', [$deploymentId]) ], $dbForProject, function (Document $document) use ($storageBuilds) { - if ($storageBuilds->delete($document->getAttribute('path', ''), true)) { - Console::success('Deleted build files: ' . $document->getAttribute('path', '')); + if ($storageBuilds->delete($document->getAttribute('outputPath', ''), true)) { + Console::success('Deleted build files: ' . $document->getAttribute('outputPath', '')); } else { - Console::error('Failed to delete build files: ' . $document->getAttribute('path', '')); + Console::error('Failed to delete build files: ' . $document->getAttribute('outputPath', '')); } }); diff --git a/app/workers/functions.php b/app/workers/functions.php index edae430fef..851defb5ca 100644 --- a/app/workers/functions.php +++ b/app/workers/functions.php @@ -93,7 +93,7 @@ Server::setResource('execute', function () { 'response' => '', 'stderr' => '', 'duration' => 0.0, - 'search' => implode(' ', [$functionId, $executionId]), + 'search' => implode(' ', [$function->getId(), $executionId]), ])); // TODO: @Meldiron Trigger executions.create event here @@ -145,7 +145,7 @@ Server::setResource('execute', function () { variables: $vars, timeout: $function->getAttribute('timeout', 0), image: $runtime['image'], - source: $build->getAttribute('path', ''), + source: $build->getAttribute('outputPath', ''), entrypoint: $deployment->getAttribute('entrypoint', ''), ); diff --git a/bin/hamster b/bin/hamster new file mode 100644 index 0000000000..dcc7ed308d --- /dev/null +++ b/bin/hamster @@ -0,0 +1,3 @@ +#!/bin/sh + +php /usr/src/code/app/cli.php hamster $@ \ No newline at end of file diff --git a/bin/patch-create-missing-schedules b/bin/patch-create-missing-schedules deleted file mode 100644 index e38d3e9a6f..0000000000 --- a/bin/patch-create-missing-schedules +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -php /usr/src/code/app/cli.php patch-create-missing-schedules $@ \ No newline at end of file diff --git a/bin/patch-delete-schedule-updated-at-attribute b/bin/patch-delete-schedule-updated-at-attribute new file mode 100644 index 0000000000..3e28289cbe --- /dev/null +++ b/bin/patch-delete-schedule-updated-at-attribute @@ -0,0 +1,3 @@ +#!/bin/sh + +php /usr/src/code/app/cli.php patch-delete-schedule-updated-at-attribute $@ \ No newline at end of file diff --git a/composer.json b/composer.json index 5ea57253c5..448243762c 100644 --- a/composer.json +++ b/composer.json @@ -44,10 +44,10 @@ "appwrite/php-clamav": "1.1.*", "appwrite/php-runtimes": "0.11.*", "utopia-php/abuse": "0.18.*", - "utopia-php/analytics": "0.2.*", "utopia-php/audit": "0.20.*", + "utopia-php/analytics": "0.10.2", "utopia-php/cache": "0.8.*", - "utopia-php/cli": "0.14.*", + "utopia-php/cli": "0.15.*", "utopia-php/config": "0.2.*", "utopia-php/database": "0.30.*", "utopia-php/queue": "0.5.*", @@ -72,7 +72,8 @@ "phpmailer/phpmailer": "6.6.0", "chillerlan/php-qrcode": "4.3.3", "adhocore/jwt": "1.1.2", - "webonyx/graphql-php": "14.11.*" + "webonyx/graphql-php": "14.11.*", + "slickdeals/statsd": "3.1.0" }, "repositories": [ { diff --git a/composer.lock b/composer.lock index 33b91c2a1e..f68956c516 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "60f1cefcf6d210ed231acb4a99f2f20c", + "content-hash": "1614e987617e0a710448662d1d9cc7ef", "packages": [ { "name": "adhocore/jwt", @@ -998,6 +998,62 @@ }, "time": "2020-04-16T16:39:50+00:00" }, + { + "name": "slickdeals/statsd", + "version": "3.1.0", + "source": { + "type": "git", + "url": "https://github.com/Slickdeals/statsd-php.git", + "reference": "225588a0a079e145359049f6e5e23eedb1b4c17f" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/Slickdeals/statsd-php/zipball/225588a0a079e145359049f6e5e23eedb1b4c17f", + "reference": "225588a0a079e145359049f6e5e23eedb1b4c17f", + "shasum": "" + }, + "require": { + "php": ">= 7.3 || ^8" + }, + "replace": { + "domnikl/statsd": "self.version" + }, + "require-dev": { + "friendsofphp/php-cs-fixer": "^3.0", + "phpunit/phpunit": "^9", + "vimeo/psalm": "^4.6" + }, + "type": "library", + "autoload": { + "psr-4": { + "Domnikl\\Statsd\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Dominik Liebler", + "email": "liebler.dominik@gmail.com" + } + ], + "description": "a PHP client for statsd", + "homepage": "https://github.com/Slickdeals/statsd-php", + "keywords": [ + "Metrics", + "monitoring", + "statistics", + "statsd", + "udp" + ], + "support": { + "issues": "https://github.com/Slickdeals/statsd-php/issues", + "source": "https://github.com/Slickdeals/statsd-php/tree/3.1.0" + }, + "time": "2021-06-04T20:33:46+00:00" + }, { "name": "symfony/polyfill-php80", "version": "v1.27.0", @@ -1133,24 +1189,25 @@ }, { "name": "utopia-php/analytics", - "version": "0.2.0", + "version": "0.10.2", "source": { "type": "git", "url": "https://github.com/utopia-php/analytics.git", - "reference": "adfc2d057a7f6ab618a77c8a20ed3e35485ff416" + "reference": "14c805114736f44c26d6d24b176a2f8b93d86a1f" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/analytics/zipball/adfc2d057a7f6ab618a77c8a20ed3e35485ff416", - "reference": "adfc2d057a7f6ab618a77c8a20ed3e35485ff416", + "url": "https://api.github.com/repos/utopia-php/analytics/zipball/14c805114736f44c26d6d24b176a2f8b93d86a1f", + "reference": "14c805114736f44c26d6d24b176a2f8b93d86a1f", "shasum": "" }, "require": { - "php": ">=7.4" + "php": ">=8.0", + "utopia-php/cli": "^0.15.0" }, "require-dev": { - "phpunit/phpunit": "^9.3", - "vimeo/psalm": "4.0.1" + "laravel/pint": "dev-main", + "phpunit/phpunit": "^9.3" }, "type": "library", "autoload": { @@ -1162,16 +1219,6 @@ "license": [ "MIT" ], - "authors": [ - { - "name": "Eldad Fux", - "email": "eldad@appwrite.io" - }, - { - "name": "Torsten Dittmann", - "email": "torsten@appwrite.io" - } - ], "description": "A simple library to track events & users.", "keywords": [ "analytics", @@ -1182,9 +1229,9 @@ ], "support": { "issues": "https://github.com/utopia-php/analytics/issues", - "source": "https://github.com/utopia-php/analytics/tree/0.2.0" + "source": "https://github.com/utopia-php/analytics/tree/0.10.2" }, - "time": "2021-03-23T21:33:07+00:00" + "time": "2023-03-22T12:01:09+00:00" }, { "name": "utopia-php/audit", @@ -1286,16 +1333,16 @@ }, { "name": "utopia-php/cli", - "version": "0.14.0", + "version": "0.15.0", "source": { "type": "git", "url": "https://github.com/utopia-php/cli.git", - "reference": "c30ef985a4e739758a0d95eb0706b357b6d8c086" + "reference": "ccb7c8125ffe0254fef8f25744bfa376eb7bd0ea" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/cli/zipball/c30ef985a4e739758a0d95eb0706b357b6d8c086", - "reference": "c30ef985a4e739758a0d95eb0706b357b6d8c086", + "url": "https://api.github.com/repos/utopia-php/cli/zipball/ccb7c8125ffe0254fef8f25744bfa376eb7bd0ea", + "reference": "ccb7c8125ffe0254fef8f25744bfa376eb7bd0ea", "shasum": "" }, "require": { @@ -1303,8 +1350,10 @@ "utopia-php/framework": "0.*.*" }, "require-dev": { + "laravel/pint": "1.2.*", "phpunit/phpunit": "^9.3", - "squizlabs/php_codesniffer": "^3.6" + "squizlabs/php_codesniffer": "^3.6", + "vimeo/psalm": "4.0.1" }, "type": "library", "autoload": { @@ -1316,12 +1365,6 @@ "license": [ "MIT" ], - "authors": [ - { - "name": "Eldad Fux", - "email": "eldad@appwrite.io" - } - ], "description": "A simple CLI library to manage command line applications", "keywords": [ "cli", @@ -1333,9 +1376,9 @@ ], "support": { "issues": "https://github.com/utopia-php/cli/issues", - "source": "https://github.com/utopia-php/cli/tree/0.14.0" + "source": "https://github.com/utopia-php/cli/tree/0.15.0" }, - "time": "2022-10-09T10:19:07+00:00" + "time": "2023-03-01T05:55:14+00:00" }, { "name": "utopia-php/config", @@ -1853,23 +1896,24 @@ }, { "name": "utopia-php/orchestration", - "version": "0.9.0", + "version": "0.9.1", "source": { "type": "git", "url": "https://github.com/utopia-php/orchestration.git", - "reference": "1d4f66684b8c4927f31b695817eae6d84aafd172" + "reference": "55f43513b3f940a3f4f9c2cde7682d0c2581beb0" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/orchestration/zipball/1d4f66684b8c4927f31b695817eae6d84aafd172", - "reference": "1d4f66684b8c4927f31b695817eae6d84aafd172", + "url": "https://api.github.com/repos/utopia-php/orchestration/zipball/55f43513b3f940a3f4f9c2cde7682d0c2581beb0", + "reference": "55f43513b3f940a3f4f9c2cde7682d0c2581beb0", "shasum": "" }, "require": { "php": ">=8.0", - "utopia-php/cli": "0.14.*" + "utopia-php/cli": "0.15.*" }, "require-dev": { + "laravel/pint": "^1.2", "phpunit/phpunit": "^9.3", "vimeo/psalm": "4.0.1" }, @@ -1896,34 +1940,34 @@ ], "support": { "issues": "https://github.com/utopia-php/orchestration/issues", - "source": "https://github.com/utopia-php/orchestration/tree/0.9.0" + "source": "https://github.com/utopia-php/orchestration/tree/0.9.1" }, - "time": "2022-11-09T17:38:00+00:00" + "time": "2023-03-17T15:05:06+00:00" }, { "name": "utopia-php/platform", - "version": "0.3.2", + "version": "0.3.3", "source": { "type": "git", "url": "https://github.com/utopia-php/platform.git", - "reference": "a53997f1882dc16ff67968aa5f436364c8c8043e" + "reference": "a9e7a501f33e0da59779782359a747cb8d34cf6f" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/platform/zipball/a53997f1882dc16ff67968aa5f436364c8c8043e", - "reference": "a53997f1882dc16ff67968aa5f436364c8c8043e", + "url": "https://api.github.com/repos/utopia-php/platform/zipball/a9e7a501f33e0da59779782359a747cb8d34cf6f", + "reference": "a9e7a501f33e0da59779782359a747cb8d34cf6f", "shasum": "" }, "require": { "ext-json": "*", "ext-redis": "*", "php": ">=8.0", - "utopia-php/cli": "0.14.*", + "utopia-php/cli": "0.15.*", "utopia-php/framework": "0.26.*" }, "require-dev": { - "phpunit/phpunit": "^9.3", - "squizlabs/php_codesniffer": "^3.6" + "laravel/pint": "1.2.*", + "phpunit/phpunit": "^9.3" }, "type": "library", "autoload": { @@ -1945,9 +1989,9 @@ ], "support": { "issues": "https://github.com/utopia-php/platform/issues", - "source": "https://github.com/utopia-php/platform/tree/0.3.2" + "source": "https://github.com/utopia-php/platform/tree/0.3.3" }, - "time": "2023-02-14T04:51:46+00:00" + "time": "2023-03-07T08:52:22+00:00" }, { "name": "utopia-php/pools", @@ -2055,21 +2099,21 @@ }, { "name": "utopia-php/queue", - "version": "0.5.1", + "version": "0.5.2", "source": { "type": "git", "url": "https://github.com/utopia-php/queue.git", - "reference": "4b39e1f232b2e619b0d7fb4004f2356df334ddc1" + "reference": "310271c5cd477541208d7fa74a4dea64df8e04a0" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/queue/zipball/4b39e1f232b2e619b0d7fb4004f2356df334ddc1", - "reference": "4b39e1f232b2e619b0d7fb4004f2356df334ddc1", + "url": "https://api.github.com/repos/utopia-php/queue/zipball/310271c5cd477541208d7fa74a4dea64df8e04a0", + "reference": "310271c5cd477541208d7fa74a4dea64df8e04a0", "shasum": "" }, "require": { "php": ">=8.0", - "utopia-php/cli": "0.14.*", + "utopia-php/cli": "0.15.*", "utopia-php/framework": "0.*.*" }, "require-dev": { @@ -2110,9 +2154,9 @@ ], "support": { "issues": "https://github.com/utopia-php/queue/issues", - "source": "https://github.com/utopia-php/queue/tree/0.5.1" + "source": "https://github.com/utopia-php/queue/tree/0.5.2" }, - "time": "2022-11-16T19:47:26+00:00" + "time": "2023-03-07T08:54:10+00:00" }, { "name": "utopia-php/registry", @@ -2564,6 +2608,49 @@ }, "time": "2023-02-03T05:44:59+00:00" }, + { + "name": "doctrine/deprecations", + "version": "v1.0.0", + "source": { + "type": "git", + "url": "https://github.com/doctrine/deprecations.git", + "reference": "0e2a4f1f8cdfc7a92ec3b01c9334898c806b30de" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/doctrine/deprecations/zipball/0e2a4f1f8cdfc7a92ec3b01c9334898c806b30de", + "reference": "0e2a4f1f8cdfc7a92ec3b01c9334898c806b30de", + "shasum": "" + }, + "require": { + "php": "^7.1|^8.0" + }, + "require-dev": { + "doctrine/coding-standard": "^9", + "phpunit/phpunit": "^7.5|^8.5|^9.5", + "psr/log": "^1|^2|^3" + }, + "suggest": { + "psr/log": "Allows logging deprecations via PSR-3 logger implementation" + }, + "type": "library", + "autoload": { + "psr-4": { + "Doctrine\\Deprecations\\": "lib/Doctrine/Deprecations" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "description": "A small layer on top of trigger_error(E_USER_DEPRECATED) or PSR-3 logging with options to disable all deprecations or selectively for packages.", + "homepage": "https://www.doctrine-project.org/", + "support": { + "issues": "https://github.com/doctrine/deprecations/issues", + "source": "https://github.com/doctrine/deprecations/tree/v1.0.0" + }, + "time": "2022-05-02T15:47:09+00:00" + }, { "name": "doctrine/instantiator", "version": "1.5.0", @@ -3096,24 +3183,27 @@ }, { "name": "phpdocumentor/type-resolver", - "version": "1.6.2", + "version": "1.7.0", "source": { "type": "git", "url": "https://github.com/phpDocumentor/TypeResolver.git", - "reference": "48f445a408c131e38cab1c235aa6d2bb7a0bb20d" + "reference": "1534aea9bde19a5c85c5d1e1f834ab63f4c5dcf5" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpDocumentor/TypeResolver/zipball/48f445a408c131e38cab1c235aa6d2bb7a0bb20d", - "reference": "48f445a408c131e38cab1c235aa6d2bb7a0bb20d", + "url": "https://api.github.com/repos/phpDocumentor/TypeResolver/zipball/1534aea9bde19a5c85c5d1e1f834ab63f4c5dcf5", + "reference": "1534aea9bde19a5c85c5d1e1f834ab63f4c5dcf5", "shasum": "" }, "require": { + "doctrine/deprecations": "^1.0", "php": "^7.4 || ^8.0", - "phpdocumentor/reflection-common": "^2.0" + "phpdocumentor/reflection-common": "^2.0", + "phpstan/phpdoc-parser": "^1.13" }, "require-dev": { "ext-tokenizer": "*", + "phpbench/phpbench": "^1.2", "phpstan/extension-installer": "^1.1", "phpstan/phpstan": "^1.8", "phpstan/phpstan-phpunit": "^1.1", @@ -3145,9 +3235,9 @@ "description": "A PSR-5 based resolver of Class names, Types and Structural Element Names", "support": { "issues": "https://github.com/phpDocumentor/TypeResolver/issues", - "source": "https://github.com/phpDocumentor/TypeResolver/tree/1.6.2" + "source": "https://github.com/phpDocumentor/TypeResolver/tree/1.7.0" }, - "time": "2022-10-14T12:47:21+00:00" + "time": "2023-03-12T10:13:29+00:00" }, { "name": "phpspec/prophecy", @@ -3217,6 +3307,51 @@ }, "time": "2023-02-02T15:41:36+00:00" }, + { + "name": "phpstan/phpdoc-parser", + "version": "1.16.1", + "source": { + "type": "git", + "url": "https://github.com/phpstan/phpdoc-parser.git", + "reference": "e27e92d939e2e3636f0a1f0afaba59692c0bf571" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/phpstan/phpdoc-parser/zipball/e27e92d939e2e3636f0a1f0afaba59692c0bf571", + "reference": "e27e92d939e2e3636f0a1f0afaba59692c0bf571", + "shasum": "" + }, + "require": { + "php": "^7.2 || ^8.0" + }, + "require-dev": { + "php-parallel-lint/php-parallel-lint": "^1.2", + "phpstan/extension-installer": "^1.0", + "phpstan/phpstan": "^1.5", + "phpstan/phpstan-phpunit": "^1.1", + "phpstan/phpstan-strict-rules": "^1.0", + "phpunit/phpunit": "^9.5", + "symfony/process": "^5.2" + }, + "type": "library", + "autoload": { + "psr-4": { + "PHPStan\\PhpDocParser\\": [ + "src/" + ] + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "description": "PHPDoc parser with support for nullable, intersection and generic types", + "support": { + "issues": "https://github.com/phpstan/phpdoc-parser/issues", + "source": "https://github.com/phpstan/phpdoc-parser/tree/1.16.1" + }, + "time": "2023-02-07T18:11:17+00:00" + }, { "name": "phpunit/php-code-coverage", "version": "9.2.26", @@ -5068,5 +5203,5 @@ "platform-overrides": { "php": "8.0" }, - "plugin-api-version": "2.2.0" + "plugin-api-version": "2.3.0" } diff --git a/docker-compose.yml b/docker-compose.yml index f137187c55..625243c1f8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -54,7 +54,7 @@ services: TESTING: true VERSION: dev VITE_CONSOLE_MODE: cloud - ports: + ports: - 9501:80 networks: - appwrite @@ -542,6 +542,40 @@ services: - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG + appwrite-hamster: + entrypoint: hamster + <<: *x-logging + container_name: appwrite-hamster + image: appwrite-dev + networks: + - appwrite + volumes: + - ./app:/usr/src/code/app + - ./src:/usr/src/code/src + depends_on: + - redis + environment: + - _APP_ENV + - _APP_WORKER_PER_CORE + - _APP_CONNECTIONS_MAX + - _APP_POOL_CLIENTS + - _APP_OPENSSL_KEY_V1 + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS + - _APP_CONNECTIONS_DB_CONSOLE + - _APP_CONNECTIONS_DB_PROJECT + - _APP_CONNECTIONS_CACHE + - _APP_HAMSTER_INTERVAL + - _APP_HAMSTER_TIME + - _APP_MIXPANEL_TOKEN + appwrite-maintenance: entrypoint: maintenance <<: *x-logging @@ -655,7 +689,7 @@ services: hostname: exc1 <<: *x-logging stop_signal: SIGINT - image: openruntimes/executor:0.2.0 + image: openruntimes/executor:0.1.6 networks: - appwrite - runtimes diff --git a/src/Appwrite/Extend/Exception.php b/src/Appwrite/Extend/Exception.php index 4f46004cea..934bf730f4 100644 --- a/src/Appwrite/Extend/Exception.php +++ b/src/Appwrite/Extend/Exception.php @@ -51,6 +51,7 @@ class Exception extends \Exception public const GENERAL_CURSOR_NOT_FOUND = 'general_cursor_not_found'; public const GENERAL_SERVER_ERROR = 'general_server_error'; public const GENERAL_PROTOCOL_UNSUPPORTED = 'general_protocol_unsupported'; + public const GENERAL_CODES_DISABLED = 'general_codes_disabled'; /** Users */ public const USER_COUNT_EXCEEDED = 'user_count_exceeded'; @@ -61,7 +62,7 @@ class Exception extends \Exception public const USER_PASSWORD_RESET_REQUIRED = 'user_password_reset_required'; public const USER_EMAIL_NOT_WHITELISTED = 'user_email_not_whitelisted'; public const USER_IP_NOT_WHITELISTED = 'user_ip_not_whitelisted'; - public const USER_CODE_INVALID = 'user_code_invalid'; + public const USER_INVALID_CODE = 'user_invalid_code'; public const USER_INVALID_CREDENTIALS = 'user_invalid_credentials'; public const USER_ANONYMOUS_CONSOLE_PROHIBITED = 'user_anonymous_console_prohibited'; public const USER_SESSION_ALREADY_EXISTS = 'user_session_already_exists'; @@ -184,6 +185,7 @@ class Exception extends \Exception public const GRAPHQL_TOO_MANY_QUERIES = 'graphql_too_many_queries'; protected $type = ''; + protected $errors = []; public function __construct(string $type = Exception::GENERAL_UNKNOWN, string $message = null, int $code = null, \Throwable $previous = null) { diff --git a/src/Appwrite/Migration/Version/V17.php b/src/Appwrite/Migration/Version/V17.php index 25f3c74598..59c57ff794 100644 --- a/src/Appwrite/Migration/Version/V17.php +++ b/src/Appwrite/Migration/Version/V17.php @@ -80,17 +80,6 @@ class V17 extends Migration } catch (\Throwable $th) { Console::warning("'mimeType' from {$id}: {$th->getMessage()}"); } - break; - case 'builds': - try { - /** - * Create 'size' attribute - */ - $this->createAttributeFromCollection($this->projectDB, $id, 'size'); - $this->projectDB->deleteCachedCollection($id); - } catch (\Throwable $th) { - Console::warning("'size' from {$id}: {$th->getMessage()}"); - } try { /** @@ -167,6 +156,16 @@ class V17 extends Migration } catch (\Throwable $th) { Console::warning("'scheduleInternalId' from {$id}: {$th->getMessage()}"); } + + try { + /** + * Delete 'scheduleUpdatedAt' attribute + */ + $this->projectDB->deleteAttribute($id, 'scheduleUpdatedAt'); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'scheduleUpdatedAt' from {$id}: {$th->getMessage()}"); + } break; case 'deployments': diff --git a/src/Appwrite/Platform/Services/Tasks.php b/src/Appwrite/Platform/Services/Tasks.php index 635722452e..927abc63a1 100644 --- a/src/Appwrite/Platform/Services/Tasks.php +++ b/src/Appwrite/Platform/Services/Tasks.php @@ -12,6 +12,9 @@ use Appwrite\Platform\Tasks\PatchCreateMissingSchedules; use Appwrite\Platform\Tasks\SDKs; use Appwrite\Platform\Tasks\Specs; use Appwrite\Platform\Tasks\SSL; +use Appwrite\Platform\Tasks\Hamster; +use Appwrite\Platform\Tasks\PatchDeleteScheduleUpdatedAtAttribute; +use Appwrite\Platform\Tasks\Usage; use Appwrite\Platform\Tasks\Vars; use Appwrite\Platform\Tasks\Version; use Appwrite\Platform\Tasks\VolumeSync; @@ -25,10 +28,12 @@ class Tasks extends Service ->addAction(Version::getName(), new Version()) ->addAction(Vars::getName(), new Vars()) ->addAction(SSL::getName(), new SSL()) + ->addAction(Hamster::getName(), new Hamster()) ->addAction(Doctor::getName(), new Doctor()) ->addAction(Install::getName(), new Install()) ->addAction(Maintenance::getName(), new Maintenance()) ->addAction(PatchCreateMissingSchedules::getName(), new PatchCreateMissingSchedules()) + ->addAction(PatchDeleteScheduleUpdatedAtAttribute::getName(), new PatchDeleteScheduleUpdatedAtAttribute()) ->addAction(Schedule::getName(), new Schedule()) ->addAction(Migrate::getName(), new Migrate()) ->addAction(SDKs::getName(), new SDKs()) diff --git a/src/Appwrite/Platform/Tasks/Hamster.php b/src/Appwrite/Platform/Tasks/Hamster.php new file mode 100644 index 0000000000..3e3cdd9d73 --- /dev/null +++ b/src/Appwrite/Platform/Tasks/Hamster.php @@ -0,0 +1,420 @@ + 'files.$all.count.total', + 'usage_buckets' => 'buckets.$all.count.total', + 'usage_databases' => 'databases.$all.count.total', + 'usage_documents' => 'documents.$all.count.total', + 'usage_collections' => 'collections.$all.count.total', + 'usage_storage' => 'project.$all.storage.size', + 'usage_requests' => 'project.$all.network.requests', + 'usage_bandwidth' => 'project.$all.network.bandwidth', + 'usage_users' => 'users.$all.count.total', + 'usage_sessions' => 'sessions.email.requests.create', + 'usage_executions' => 'executions.$all.compute.total', + ]; + + protected string $directory = '/usr/local'; + + protected string $path; + + protected string $date; + + protected Mixpanel $mixpanel; + + public static function getName(): string + { + return 'hamster'; + } + + public function __construct() + { + $this->mixpanel = new Mixpanel(App::getEnv('_APP_MIXPANEL_TOKEN', '')); + + $this + ->desc('Get stats for projects') + ->inject('pools') + ->inject('cache') + ->inject('dbForConsole') + ->callback(function (Group $pools, Cache $cache, Database $dbForConsole) { + $this->action($pools, $cache, $dbForConsole); + }); + } + + private function getStatsPerProject(Group $pools, Cache $cache, Database $dbForConsole) + { + $this->calculateByGroup('projects', $dbForConsole, function (Database $dbForConsole, Document $project) use ($pools, $cache) { + /** + * Skip user projects with id 'console' + */ + if ($project->getId() === 'console') { + Console::info("Skipping project console"); + return; + } + + Console::log("Getting stats for {$project->getId()}"); + + try { + $db = $project->getAttribute('database'); + $adapter = $pools + ->get($db) + ->pop() + ->getResource(); + + $dbForProject = new Database($adapter, $cache); + $dbForProject->setDefaultDatabase('appwrite'); + $dbForProject->setNamespace('_' . $project->getInternalId()); + + $statsPerProject = []; + + $statsPerProject['time'] = microtime(true); + + /** Get Project ID */ + $statsPerProject['project_id'] = $project->getId(); + + /** Get project created time */ + $statsPerProject['project_created'] = $project->getAttribute('$createdAt'); + + /** Get Project Name */ + $statsPerProject['project_name'] = $project->getAttribute('name'); + + /** Get Total Functions */ + $statsPerProject['custom_functions'] = $dbForProject->count('functions', [], APP_LIMIT_COUNT); + + /** Get Total Deployments */ + $statsPerProject['custom_deployments'] = $dbForProject->count('deployments', [], APP_LIMIT_COUNT); + + /** Get Total Teams */ + $statsPerProject['custom_teams'] = $dbForProject->count('teams', [], APP_LIMIT_COUNT); + + /** Get Total Members */ + $teamInternalId = $project->getAttribute('teamInternalId', null); + if ($teamInternalId) { + $statsPerProject['custom_organization_members'] = $dbForConsole->count('memberships', [ + Query::equal('teamInternalId', [$teamInternalId]) + ], APP_LIMIT_COUNT); + } else { + $statsPerProject['custom_organization_members'] = 0; + } + + /** Get Email and Name of the project owner */ + if ($teamInternalId) { + $membership = $dbForConsole->findOne('memberships', [ + Query::equal('teamInternalId', [$teamInternalId]), + ]); + + if (!$membership || $membership->isEmpty()) { + throw new Exception('Membership not found. Skipping project : ' . $project->getId()); + } + + $userInternalId = $membership->getAttribute('userInternalId', null); + if ($userInternalId) { + $user = $dbForConsole->findOne('users', [ + Query::equal('_id', [$userInternalId]), + ]); + + $statsPerProject['email'] = $user->getAttribute('email', null); + $statsPerProject['name'] = $user->getAttribute('name', null); + } + } + + /** Get Domains */ + $statsPerProject['custom_domains'] = $dbForProject->count('domains', [], APP_LIMIT_COUNT); + + /** Get Platforms */ + $platforms = $dbForConsole->find('platforms', [ + Query::equal('projectInternalId', [$project->getInternalId()]), + Query::limit(APP_LIMIT_COUNT) + ]); + + $statsPerProject['custom_platforms_web'] = sizeof(array_filter($platforms, function ($platform) { + return $platform['type'] === 'web'; + })); + + $statsPerProject['custom_platforms_android'] = sizeof(array_filter($platforms, function ($platform) { + return $platform['type'] === 'android'; + })); + + $statsPerProject['custom_platforms_iOS'] = sizeof(array_filter($platforms, function ($platform) { + return str_contains($platform['type'], 'apple'); + })); + + $statsPerProject['custom_platforms_flutter'] = sizeof(array_filter($platforms, function ($platform) { + return str_contains($platform['type'], 'flutter'); + })); + + /** Get Usage $statsPerProject */ + $periods = [ + 'infinity' => [ + 'period' => '1d', + 'limit' => 90, + ], + '24h' => [ + 'period' => '1h', + 'limit' => 24, + ], + ]; + + Authorization::skip(function () use ($dbForProject, $periods, &$statsPerProject) { + foreach ($this->metrics as $key => $metric) { + foreach ($periods as $periodKey => $periodValue) { + $limit = $periodValue['limit']; + $period = $periodValue['period']; + + $requestDocs = $dbForProject->find('stats', [ + Query::equal('period', [$period]), + Query::equal('metric', [$metric]), + Query::limit($limit), + Query::orderDesc('time'), + ]); + + $statsPerProject[$key . '_' . $periodKey] = []; + foreach ($requestDocs as $requestDoc) { + $statsPerProject[$key . '_' . $periodKey][] = [ + 'value' => $requestDoc->getAttribute('value'), + 'date' => $requestDoc->getAttribute('time'), + ]; + } + + $statsPerProject[$key . '_' . $periodKey] = array_reverse($statsPerProject[$key . '_' . $periodKey]); + // Calculate aggregate of each metric + $statsPerProject[$key . '_' . $periodKey] = array_sum(array_column($statsPerProject[$key . '_' . $periodKey], 'value')); + } + } + }); + + if (isset($statsPerProject['email'])) { + /** Send data to mixpanel */ + $res = $this->mixpanel->createProfile($statsPerProject['email'], '', [ + 'name' => $statsPerProject['name'], + 'email' => $statsPerProject['email'] + ]); + + if (!$res) { + Console::error('Failed to create user profile for project: ' . $project->getId()); + } + + $event = new Event(); + $event + ->setName('Project Daily Usage') + ->setProps($statsPerProject); + $res = $this->mixpanel->createEvent($event); + if (!$res) { + Console::error('Failed to create event for project: ' . $project->getId()); + } + } + } catch (Exception $e) { + Console::error('Failed to send stats for project: ' . $project->getId()); + Console::error($e->getMessage()); + } finally { + $pools + ->get($db) + ->reclaim(); + } + }); + } + + public function action(Group $pools, Cache $cache, Database $dbForConsole): void + { + + Console::title('Cloud Hamster V1'); + Console::success(APP_NAME . ' cloud hamster process has started'); + + $sleep = (int) App::getEnv('_APP_HAMSTER_INTERVAL', '30'); // 30 seconds (by default) + + $jobInitTime = App::getEnv('_APP_HAMSTER_TIME', '22:00'); // (hour:minutes) + $now = new \DateTime(); + $now->setTimezone(new \DateTimeZone(date_default_timezone_get())); + $next = new \DateTime($now->format("Y-m-d $jobInitTime")); + $next->setTimezone(new \DateTimeZone(date_default_timezone_get())); + $delay = $next->getTimestamp() - $now->getTimestamp(); + + /** + * If time passed for the target day. + */ + if ($delay <= 0) { + $next->add(\DateInterval::createFromDateString('1 days')); + $delay = $next->getTimestamp() - $now->getTimestamp(); + } + + Console::log('[' . $now->format("Y-m-d H:i:s.v") . '] Delaying for ' . $delay . ' setting loop to [' . $next->format("Y-m-d H:i:s.v") . ']'); + + Console::loop(function () use ($pools, $cache, $dbForConsole, $sleep) { + $now = date('d-m-Y H:i:s', time()); + Console::info("[{$now}] Getting Cloud Usage Stats every {$sleep} seconds"); + $loopStart = microtime(true); + + /* Initialise new Utopia app */ + $app = new App('UTC'); + + Console::info('Getting stats for all projects'); + $this->getStatsPerProject($pools, $cache, $dbForConsole); + Console::success('Completed getting stats for all projects'); + + Console::info('Getting stats for all organizations'); + $this->getStatsPerOrganization($dbForConsole); + Console::success('Completed getting stats for all organizations'); + + Console::info('Getting stats for all users'); + $this->getStatsPerUser($dbForConsole); + Console::success('Completed getting stats for all users'); + + $pools + ->get('console') + ->reclaim(); + + $loopTook = microtime(true) - $loopStart; + $now = date('d-m-Y H:i:s', time()); + Console::info("[{$now}] Cloud Stats took {$loopTook} seconds"); + }, $sleep, $delay); + } + + protected function calculateByGroup(string $collection, Database $dbForConsole, callable $callback) + { + $count = 0; + $chunk = 0; + $limit = 50; + $results = []; + $sum = $limit; + + $executionStart = \microtime(true); + + while ($sum === $limit) { + $chunk++; + + $results = $dbForConsole->find($collection, \array_merge([ + Query::limit($limit), + Query::offset($count) + ])); + + $sum = count($results); + + Console::log('Processing chunk #' . $chunk . '. Found ' . $sum . ' documents'); + + foreach ($results as $document) { + call_user_func($callback, $dbForConsole, $document); + $count++; + } + } + + $executionEnd = \microtime(true); + + Console::log("Processed {$count} document by group in " . ($executionEnd - $executionStart) . " seconds"); + } + + protected function getStatsPerOrganization(Database $dbForConsole) + { + + $this->calculateByGroup('teams', $dbForConsole, function (Database $dbForConsole, Document $document) { + try { + $statsPerOrganization = []; + + /** Organization name */ + $statsPerOrganization['name'] = $document->getAttribute('name'); + + /** Get Email and of the organization owner */ + $membership = $dbForConsole->findOne('memberships', [ + Query::equal('teamInternalId', [$document->getInternalId()]), + ]); + + if (!$membership || $membership->isEmpty()) { + throw new Exception('Membership not found. Skipping organization : ' . $document->getId()); + } + + $userInternalId = $membership->getAttribute('userInternalId', null); + if ($userInternalId) { + $user = $dbForConsole->findOne('users', [ + Query::equal('_id', [$userInternalId]), + ]); + + $statsPerOrganization['email'] = $user->getAttribute('email', null); + } + + /** Organization Creation Date */ + $statsPerOrganization['created'] = $document->getAttribute('$createdAt'); + + /** Number of team members */ + $statsPerOrganization['members'] = $document->getAttribute('total'); + + /** Number of projects in this organization */ + $statsPerOrganization['projects'] = $dbForConsole->count('projects', [ + Query::equal('teamId', [$document->getId()]), + Query::limit(APP_LIMIT_COUNT) + ]); + + if (!isset($statsPerOrganization['email'])) { + throw new Exception('Email not found. Skipping organization : ' . $document->getId()); + } + + $event = new Event(); + $event + ->setName('Organization Daily Usage') + ->setProps($statsPerOrganization); + $res = $this->mixpanel->createEvent($event); + if (!$res) { + throw new Exception('Failed to create event for organization : ' . $document->getId()); + } + } catch (Exception $e) { + Console::error($e->getMessage()); + } + }); + } + + protected function getStatsPerUser(Database $dbForConsole) + { + $this->calculateByGroup('users', $dbForConsole, function (Database $dbForConsole, Document $document) { + try { + $statsPerUser = []; + + /** Organization name */ + $statsPerUser['name'] = $document->getAttribute('name'); + + /** Organization ID (needs to be stored as an email since mixpanel uses the email attribute as a distinctID) */ + $statsPerUser['email'] = $document->getAttribute('email'); + + /** Organization Creation Date */ + $statsPerUser['created'] = $document->getAttribute('$createdAt'); + + /** Number of teams this user is a part of */ + $statsPerUser['memberships'] = $dbForConsole->count('memberships', [ + Query::equal('userInternalId', [$document->getInternalId()]), + Query::limit(APP_LIMIT_COUNT) + ]); + + if (!isset($statsPerUser['email'])) { + throw new Exception('User has no email: ' . $document->getId()); + } + + /** Send data to mixpanel */ + $event = new Event(); + $event + ->setName('User Daily Usage') + ->setProps($statsPerUser); + $res = $this->mixpanel->createEvent($event); + + if (!$res) { + throw new Exception('Failed to create user profile for user: ' . $document->getId()); + } + } catch (Exception $e) { + Console::error($e->getMessage()); + } + }); + } +} diff --git a/src/Appwrite/Platform/Tasks/PatchDeleteScheduleUpdatedAtAttribute.php b/src/Appwrite/Platform/Tasks/PatchDeleteScheduleUpdatedAtAttribute.php new file mode 100644 index 0000000000..95a7c4ffe1 --- /dev/null +++ b/src/Appwrite/Platform/Tasks/PatchDeleteScheduleUpdatedAtAttribute.php @@ -0,0 +1,74 @@ +desc('Ensure function collections do not have scheduleUpdatedAt attribute') + ->inject('pools') + ->inject('dbForConsole') + ->inject('getProjectDB') + ->callback(fn (Group $pools, Database $dbForConsole, callable $getProjectDB) => $this->action($pools, $dbForConsole, $getProjectDB)); + } + + /** + * Iterate over every function on every project to make sure there is a schedule. If not, recreate the schedule. + */ + public function action(Group $pools, Database $dbForConsole, callable $getProjectDB): void + { + Authorization::disable(); + Authorization::setDefaultStatus(false); + + Console::title('PatchDeleteScheduleUpdatedAtAttribute V1'); + Console::success(APP_NAME . ' PatchDeleteScheduleUpdatedAtAttribute v1 has started'); + + $limit = 100; + $projectCursor = null; + while (true) { + $projectsQueries = [Query::limit($limit)]; + if ($projectCursor !== null) { + $projectsQueries[] = Query::cursorAfter($projectCursor); + } + $projects = $dbForConsole->find('projects', $projectsQueries); + + if (count($projects) === 0) { + break; + } + + foreach ($projects as $project) { + Console::log("Checking Project " . $project->getAttribute('name') . " (" . $project->getId() . ")"); + $dbForProject = $getProjectDB($project); + + try { + /** + * Delete 'scheduleUpdatedAt' attribute + */ + $dbForProject->deleteAttribute('functions', 'scheduleUpdatedAt'); + $dbForProject->deleteCachedCollection('functions'); + Console::success("'scheduleUpdatedAt' deleted."); + } catch (\Throwable $th) { + Console::warning("'scheduleUpdatedAt' errored: {$th->getMessage()}"); + } + + $pools->reclaim(); + } + + $projectCursor = $projects[array_key_last($projects)]; + } + } +} diff --git a/src/Appwrite/Utopia/Response/Model/Build.php b/src/Appwrite/Utopia/Response/Model/Build.php index 4d7a6cd27c..b76f0ee083 100644 --- a/src/Appwrite/Utopia/Response/Model/Build.php +++ b/src/Appwrite/Utopia/Response/Model/Build.php @@ -51,18 +51,18 @@ class Build extends Model 'default' => '', 'example' => self::TYPE_DATETIME_EXAMPLE, ]) + ->addRule('endTime', [ + 'type' => self::TYPE_DATETIME, + 'description' => 'The time the build was finished in ISO 8601 format.', + 'default' => '', + 'example' => self::TYPE_DATETIME_EXAMPLE, + ]) ->addRule('duration', [ 'type' => self::TYPE_INTEGER, 'description' => 'The build duration in seconds.', 'default' => 0, 'example' => 0, ]) - ->addRule('size', [ - 'type' => self::TYPE_INTEGER, - 'description' => 'The code size in bytes.', - 'default' => 0, - 'example' => 128, - ]) ; } diff --git a/tests/e2e/Services/Account/AccountConsoleClientTest.php b/tests/e2e/Services/Account/AccountConsoleClientTest.php index 027369d27d..c8ebedd991 100644 --- a/tests/e2e/Services/Account/AccountConsoleClientTest.php +++ b/tests/e2e/Services/Account/AccountConsoleClientTest.php @@ -39,7 +39,7 @@ class AccountConsoleClientTest extends Scope ]); $this->assertEquals($response['headers']['status-code'], 401); - $this->assertEquals($response['body']['type'], Exception::USER_CODE_INVALID); + $this->assertEquals($response['body']['type'], Exception::USER_INVALID_CODE); $response = $this->client->call(Client::METHOD_POST, '/account/invite', array_merge([ 'origin' => 'http://localhost', @@ -53,7 +53,7 @@ class AccountConsoleClientTest extends Scope ]); $this->assertEquals($response['headers']['status-code'], 401); - $this->assertEquals($response['body']['type'], Exception::USER_CODE_INVALID); + $this->assertEquals($response['body']['type'], Exception::USER_INVALID_CODE); /** * Test for SUCCESS