diff --git a/.travis.yml b/.travis.yml index a90890423..9279ed15b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,6 +6,10 @@ arch: os: linux +# Small change +vm: + size: large + language: shell notifications: @@ -35,6 +39,12 @@ install: script: - docker ps -a +# Tests should fail if any container is in exited status +- ALL_UP=`docker ps -aq --filter "status=exited"` +- > + if [[ "$ALL_UP" != "" ]]; then + exit 1 + fi - docker-compose logs appwrite - docker-compose logs mariadb - docker-compose logs appwrite-worker-functions diff --git a/Dockerfile b/Dockerfile index 611bcfaf3..ebf2dd8b1 100755 --- a/Dockerfile +++ b/Dockerfile @@ -225,6 +225,7 @@ RUN mkdir -p /storage/uploads && \ # Executables RUN chmod +x /usr/local/bin/doctor && \ chmod +x /usr/local/bin/maintenance && \ + chmod +x /usr/local/bin/usage && \ chmod +x /usr/local/bin/install && \ chmod +x /usr/local/bin/migrate && \ chmod +x /usr/local/bin/schedule && \ diff --git a/app/cli.php b/app/cli.php index ee6fe1a80..99a77d638 100644 --- a/app/cli.php +++ b/app/cli.php @@ -1,6 +1,6 @@ task('version') diff --git a/app/config/collections2.php b/app/config/collections2.php index e4f5c0147..c3e9bea46 100644 --- a/app/config/collections2.php +++ b/app/config/collections2.php @@ -510,8 +510,8 @@ $collections = [ 'signed' => true, 'required' => false, 'default' => null, - 'array' => true, - 'filters' => ['json'], + 'array' => false, + 'filters' => ['subQueryPlatforms'], ], [ '$id' => 'webhooks', @@ -521,8 +521,8 @@ $collections = [ 'signed' => true, 'required' => false, 'default' => null, - 'array' => true, - 'filters' => ['json'], + 'array' => false, + 'filters' => ['subQueryWebhooks'], ], [ '$id' => 'keys', @@ -532,8 +532,8 @@ $collections = [ 'signed' => true, 'required' => false, 'default' => null, - 'array' => true, - 'filters' => ['json'], + 'array' => false, + 'filters' => ['subQueryKeys'], ], [ '$id' => 'domains', @@ -543,16 +543,381 @@ $collections = [ 'signed' => true, 'required' => false, 'default' => null, - 'array' => true, - 'filters' => ['json'], + 'array' => false, + 'filters' => ['subQueryDomains'], + ], + [ + '$id' => 'search', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 16384, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], ], ], 'indexes' => [ [ - '$id' => '_fulltext_name', + '$id' => '_key_search', 'type' => Database::INDEX_FULLTEXT, - 'attributes' => ['name'], - 'lengths' => [1024], + 'attributes' => ['search'], + 'lengths' => [2048], + 'orders' => [Database::ORDER_ASC], + ], + ], + ], + + 'platforms' => [ + '$collection' => Database::METADATA, + '$id' => 'platforms', + 'name' => 'platforms', + 'attributes' => [ + [ + '$id' => 'projectId', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'type', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 16, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'name', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 256, + 'signed' => true, + 'required' => true, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'key', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'store', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 256, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'hostname', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 256, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'dateCreated', + 'type' => Database::VAR_INTEGER, + 'format' => '', + 'size' => 0, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'dateUpdated', + 'type' => Database::VAR_INTEGER, + 'format' => '', + 'size' => 0, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + ], + 'indexes' => [ + [ + '$id' => '_key_project', + 'type' => Database::INDEX_KEY, + 'attributes' => ['projectId'], + 'lengths' => [Database::LENGTH_KEY], + 'orders' => [Database::ORDER_ASC], + ], + ], + ], + + 'domains' => [ + '$collection' => Database::METADATA, + '$id' => 'domains', + 'name' => 'domains', + 'attributes' => [ + [ + '$id' => 'projectId', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'updated', + 'type' => Database::VAR_INTEGER, + 'format' => '', + 'size' => 0, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'domain', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => true, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'tld', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => true, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'registerable', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => true, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'verification', + 'type' => Database::VAR_BOOLEAN, + 'format' => '', + 'size' => 0, + 'signed' => true, + 'required' => true, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'certificateId', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + ], + 'indexes' => [ + [ + '$id' => '_key_project', + 'type' => Database::INDEX_KEY, + 'attributes' => ['projectId'], + 'lengths' => [Database::LENGTH_KEY], + 'orders' => [Database::ORDER_ASC], + ], + ], + ], + + 'keys' => [ + '$collection' => Database::METADATA, + '$id' => 'keys', + 'name' => 'keys', + 'attributes' => [ + [ + '$id' => 'projectId', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'name', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => true, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'scopes', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => true, + 'default' => null, + 'array' => true, + 'filters' => [], + ], + [ + '$id' => 'secret', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 256, // var_dump of \bin2hex(\random_bytes(128)) => string(256) + 'signed' => true, + 'required' => true, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + ], + 'indexes' => [ + [ + '$id' => '_key_project', + 'type' => Database::INDEX_KEY, + 'attributes' => ['projectId'], + 'lengths' => [Database::LENGTH_KEY], + 'orders' => [Database::ORDER_ASC], + ], + ], + ], + + 'webhooks' => [ + '$collection' => Database::METADATA, + '$id' => 'webhooks', + 'name' => 'webhooks', + 'attributes' => [ + [ + '$id' => 'projectId', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'name', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => true, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'url', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => true, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'httpUser', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => true, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'httpPass', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => true, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'security', + 'type' => Database::VAR_BOOLEAN, + 'format' => '', + 'size' => 0, + 'signed' => true, + 'required' => true, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'events', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => true, + 'default' => null, + 'array' => true, + 'filters' => [], + ], + ], + 'indexes' => [ + [ + '$id' => '_key_project', + 'type' => Database::INDEX_KEY, + 'attributes' => ['projectId'], + 'lengths' => [Database::LENGTH_KEY], 'orders' => [Database::ORDER_ASC], ], ], @@ -578,7 +943,7 @@ $collections = [ '$id' => 'email', 'type' => Database::VAR_STRING, 'format' => '', - 'size' => 1024, + 'size' => 320, 'signed' => true, 'required' => false, 'default' => null, @@ -695,15 +1060,51 @@ $collections = [ 'array' => true, 'filters' => ['json'], ], + [ + '$id' => 'search', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 16384, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'deleted', + 'type' => Database::VAR_BOOLEAN, + 'format' => '', + 'size' => 0, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], ], 'indexes' => [ [ '$id' => '_key_email', 'type' => Database::INDEX_UNIQUE, 'attributes' => ['email'], - 'lengths' => [1024], + 'lengths' => [320], 'orders' => [Database::ORDER_ASC], ], + [ + '$id' => '_key_search', + 'type' => Database::INDEX_FULLTEXT, + 'attributes' => ['search'], + 'lengths' => [2048], + 'orders' => [Database::ORDER_ASC], + ], + [ + '$id' => '_key_deleted_email', + 'type' => Database::INDEX_KEY, + 'attributes' => ['deleted', 'email'], + 'lengths' => [0, 320], + 'orders' => [Database::ORDER_ASC, Database::ORDER_ASC], + ], ], ], @@ -993,13 +1394,24 @@ $collections = [ 'array' => false, 'filters' => [], ], + [ + '$id' => 'search', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 16384, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], ], 'indexes' => [ [ - '$id' => '_fulltext_name', + '$id' => '_key_search', 'type' => Database::INDEX_FULLTEXT, - 'attributes' => ['name'], - 'lengths' => [1024], + 'attributes' => ['search'], + 'lengths' => [2048], 'orders' => [Database::ORDER_ASC], ], ], @@ -1273,6 +1685,17 @@ $collections = [ 'array' => false, 'filters' => [], ], + [ + '$id' => 'search', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 16384, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], ], 'indexes' => [ [ @@ -1283,10 +1706,10 @@ $collections = [ 'orders' => [Database::ORDER_ASC], ], [ - '$id' => '_fulltext_name', + '$id' => '_key_search', 'type' => Database::INDEX_FULLTEXT, - 'attributes' => ['name'], - 'lengths' => [1024], + 'attributes' => ['search'], + 'lengths' => [2048], 'orders' => [Database::ORDER_ASC], ], ], @@ -1441,13 +1864,24 @@ $collections = [ 'array' => false, 'filters' => [], ], + [ + '$id' => 'search', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 16384, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], ], 'indexes' => [ [ - '$id' => '_fulltext_name', + '$id' => '_key_search', 'type' => Database::INDEX_FULLTEXT, - 'attributes' => ['name'], - 'lengths' => [1024], + 'attributes' => ['search'], + 'lengths' => [2048], 'orders' => [Database::ORDER_ASC], ], ], @@ -1514,6 +1948,17 @@ $collections = [ 'array' => false, 'filters' => [], ], + [ + '$id' => 'search', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 16384, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], ], 'indexes' => [ [ @@ -1523,6 +1968,13 @@ $collections = [ 'lengths' => [Database::LENGTH_KEY], 'orders' => [Database::ORDER_ASC], ], + [ + '$id' => '_key_search', + 'type' => Database::INDEX_FULLTEXT, + 'attributes' => ['search'], + 'lengths' => [2048], + 'orders' => [Database::ORDER_ASC], + ], ], ], @@ -1631,6 +2083,17 @@ $collections = [ 'array' => false, 'filters' => [], ], + [ + '$id' => 'search', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 16384, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], ], 'indexes' => [ [ @@ -1640,6 +2103,13 @@ $collections = [ 'lengths' => [Database::LENGTH_KEY], 'orders' => [Database::ORDER_ASC], ], + [ + '$id' => '_fulltext_search', + 'type' => Database::INDEX_FULLTEXT, + 'attributes' => ['search'], + 'lengths' => [16384], + 'orders' => [Database::ORDER_ASC], + ], ], ], @@ -1728,6 +2198,91 @@ $collections = [ ], ], ], + 'stats' => [ + '$collection' => Database::METADATA, + '$id' => 'stats', + 'name' => 'Stats', + 'attributes' => [ + [ + '$id' => 'metric', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 255, + 'signed' => true, + 'required' => true, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'value', + 'type' => Database::VAR_INTEGER, + 'format' => '', + 'size' => 0, + 'signed' => false, + 'required' => true, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'time', + 'type' => Database::VAR_INTEGER, + 'format' => '', + 'size' => 0, + 'signed' => false, + 'required' => true, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'period', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 4, + 'signed' => true, + 'required' => true, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'type', + 'type' => Database::VAR_INTEGER, + 'format' => '', + 'size' => 1, + 'signed' => false, + 'required' => true, + 'default' => 0, // 0 -> count, 1 -> sum + 'array' => false, + 'filters' => [], + ], + ], + 'indexes' => [ + [ + '$id' => '_key_time', + 'type' => Database::INDEX_KEY, + 'attributes' => ['time'], + 'lengths' => [], + 'orders' => [Database::ORDER_DESC], + ], + [ + '$id' => '_key_metric', + 'type' => Database::INDEX_KEY, + 'attributes' => ['metric'], + 'lengths' => [], + 'orders' => [Database::ORDER_ASC], + ], + [ + '$id' => '_key_metric_period', + 'type' => Database::INDEX_KEY, + 'attributes' => ['metric', 'period'], + 'lengths' => [], + 'orders' => [Database::ORDER_DESC], + ], + ], + ] ]; -return $collections; +return $collections; \ No newline at end of file diff --git a/app/config/services.php b/app/config/services.php index 09ea35f74..17375c360 100644 --- a/app/config/services.php +++ b/app/config/services.php @@ -5,6 +5,7 @@ return [ 'key' => 'homepage', 'name' => 'Homepage', 'subtitle' => '', + 'description' => '', 'controller' => 'web/home.php', 'sdk' => false, 'docs' => false, @@ -16,6 +17,8 @@ return [ 'console' => [ 'key' => 'console', 'name' => 'Console', + 'subtitle' => '', + 'description' => '', 'controller' => 'web/console.php', 'sdk' => false, 'docs' => false, @@ -93,6 +96,7 @@ return [ 'key' => 'projects', 'name' => 'Projects', 'subtitle' => 'The Project service allows you to manage all the projects in your Appwrite server.', + 'description' => '', 'controller' => 'api/projects.php', 'sdk' => true, 'docs' => true, diff --git a/app/config/variables.php b/app/config/variables.php index df7346cd5..132e279c2 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -149,6 +149,15 @@ return [ 'required' => false, 'question' => '', 'filter' => '' + ], + [ + 'name' => '_APP_USAGE_AGGREGATION_INTERVAL', + 'description' => 'Interval value containing the number of seconds that the Appwrite usage process should wait before aggregating stats and syncing it to mariadb from InfluxDB. The default value is 30 seconds.', + 'introduction' => '0.10.0', + 'default' => '30', + 'required' => false, + 'question' => '', + 'filter' => '' ] ], ], diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index d0d9046be..7619301d1 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -52,12 +52,14 @@ App::post('/v1/account') ->inject('project') ->inject('dbForInternal') ->inject('audits') - ->action(function ($userId, $email, $password, $name, $request, $response, $project, $dbForInternal, $audits) { + ->inject('usage') + ->action(function ($userId, $email, $password, $name, $request, $response, $project, $dbForInternal, $audits, $usage) { /** @var Utopia\Swoole\Request $request */ /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Document $project */ /** @var Utopia\Database\Database $dbForInternal */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ $email = \strtolower($email); if ('console' === $project->getId()) { @@ -76,7 +78,9 @@ App::post('/v1/account') $limit = $project->getAttribute('auths', [])['limit'] ?? 0; if ($limit !== 0) { - $sum = $dbForInternal->count('users', [], APP_LIMIT_USERS); + $sum = $dbForInternal->count('users', [ + new Query('deleted', Query::TYPE_EQUAL, [false]), + ], APP_LIMIT_USERS); if ($sum >= $limit) { throw new Exception('Project registration is restricted. Contact your administrator for more information.', 501); @@ -103,6 +107,8 @@ App::post('/v1/account') 'sessions' => [], 'tokens' => [], 'memberships' => [], + 'search' => implode(' ', [$userId, $email, $name]), + 'deleted' => false ])); } catch (Duplicate $th) { throw new Exception('Account already exists', 409); @@ -120,6 +126,9 @@ App::post('/v1/account') ->setParam('resource', 'user/' . $user->getId()) ; + $usage + ->setParam('users.create', 1) + ; $response->setStatusCode(Response::STATUS_CODE_CREATED); $response->dynamic($user, Response::MODEL_USER); }); @@ -147,18 +156,20 @@ App::post('/v1/account/sessions') ->inject('locale') ->inject('geodb') ->inject('audits') - ->action(function ($email, $password, $request, $response, $dbForInternal, $locale, $geodb, $audits) { + ->inject('usage') + ->action(function ($email, $password, $request, $response, $dbForInternal, $locale, $geodb, $audits, $usage) { /** @var Utopia\Swoole\Request $request */ /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ /** @var Utopia\Locale\Locale $locale */ /** @var MaxMind\Db\Reader $geodb */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ $email = \strtolower($email); $protocol = $request->getProtocol(); - $profile = $dbForInternal->findOne('users', [new Query('email', Query::TYPE_EQUAL, [$email])]); // Get user by email address + $profile = $dbForInternal->findOne('users', [new Query('deleted', Query::TYPE_EQUAL, [false]), new Query('email', Query::TYPE_EQUAL, [$email])]); // Get user by email address if (!$profile || !Auth::passwordVerify($password, $profile->getAttribute('password'))) { $audits @@ -195,8 +206,8 @@ App::post('/v1/account/sessions') Authorization::setRole('user:' . $profile->getId()); $session = $dbForInternal->createDocument('sessions', $session - ->setAttribute('$read', ['user:' . $profile->getId()]) - ->setAttribute('$write', ['user:' . $profile->getId()]) + ->setAttribute('$read', ['user:' . $profile->getId()]) + ->setAttribute('$write', ['user:' . $profile->getId()]) ); $profile->setAttribute('sessions', $session, Document::SET_TYPE_APPEND); @@ -227,6 +238,11 @@ App::post('/v1/account/sessions') ->setAttribute('countryName', $countryName) ; + $usage + ->setParam('users.update', 1) + ->setParam('users.sessions.create', 1) + ->setParam('provider', 'email') + ; $response->dynamic($session, Response::MODEL_SESSION); }); @@ -357,7 +373,8 @@ App::get('/v1/account/sessions/oauth2/:provider/redirect') ->inject('geodb') ->inject('audits') ->inject('events') - ->action(function ($provider, $code, $state, $request, $response, $project, $user, $dbForInternal, $geodb, $audits, $events) use ($oauthDefaultSuccess) { + ->inject('usage') + ->action(function ($provider, $code, $state, $request, $response, $project, $user, $dbForInternal, $geodb, $audits, $events, $usage) use ($oauthDefaultSuccess) { /** @var Utopia\Swoole\Request $request */ /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Document $project */ @@ -365,6 +382,7 @@ App::get('/v1/account/sessions/oauth2/:provider/redirect') /** @var Utopia\Database\Database $dbForInternal */ /** @var MaxMind\Db\Reader $geodb */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ $protocol = $request->getProtocol(); $callback = $protocol . '://' . $request->getHostname() . '/v1/account/sessions/oauth2/callback/' . $provider . '/' . $project->getId(); @@ -448,13 +466,13 @@ App::get('/v1/account/sessions/oauth2/:provider/redirect') $name = $oauth2->getUserName($accessToken); $email = $oauth2->getUserEmail($accessToken); - $user = $dbForInternal->findOne('users', [new Query('email', Query::TYPE_EQUAL, [$email])]); // Get user by email address + $user = $dbForInternal->findOne('users', [new Query('deleted', Query::TYPE_EQUAL, [false]), new Query('email', Query::TYPE_EQUAL, [$email])]); // Get user by email address if ($user === false || $user->isEmpty()) { // Last option -> create the user, generate random password $limit = $project->getAttribute('auths', [])['limit'] ?? 0; if ($limit !== 0) { - $sum = $dbForInternal->count('users', [], APP_LIMIT_COUNT); + $sum = $dbForInternal->count('users', [ new Query('deleted', Query::TYPE_EQUAL, [false]),], APP_LIMIT_COUNT); if ($sum >= $limit) { throw new Exception('Project registration is restricted. Contact your administrator for more information.', 501); @@ -481,6 +499,8 @@ App::get('/v1/account/sessions/oauth2/:provider/redirect') 'sessions' => [], 'tokens' => [], 'memberships' => [], + 'search' => implode(' ', [$userId, $email, $name]), + 'deleted' => false ])); } catch (Duplicate $th) { throw new Exception('Account already exists', 409); @@ -530,8 +550,8 @@ App::get('/v1/account/sessions/oauth2/:provider/redirect') Authorization::setRole('user:' . $user->getId()); $session = $dbForInternal->createDocument('sessions', $session - ->setAttribute('$read', ['user:' . $user->getId()]) - ->setAttribute('$write', ['user:' . $user->getId()]) + ->setAttribute('$read', ['user:' . $user->getId()]) + ->setAttribute('$write', ['user:' . $user->getId()]) ); $user = $dbForInternal->updateDocument('users', $user->getId(), $user); @@ -545,6 +565,11 @@ App::get('/v1/account/sessions/oauth2/:provider/redirect') $events->setParam('eventData', $response->output($session, Response::MODEL_SESSION)); + $usage + ->setParam('users.sessions.create', 1) + ->setParam('projectId', $project->getId()) + ->setParam('provider', 'oauth2-'.$provider) + ; if (!Config::getParam('domainVerification')) { $response ->addHeader('X-Fallback-Cookies', \json_encode([Auth::$cookieName => Auth::encodeSession($user->getId(), $secret)])) @@ -595,7 +620,8 @@ App::post('/v1/account/sessions/anonymous') ->inject('dbForInternal') ->inject('geodb') ->inject('audits') - ->action(function ($request, $response, $locale, $user, $project, $dbForInternal, $geodb, $audits) { + ->inject('usage') + ->action(function ($request, $response, $locale, $user, $project, $dbForInternal, $geodb, $audits, $usage) { /** @var Utopia\Swoole\Request $request */ /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Locale\Locale $locale */ @@ -604,6 +630,7 @@ App::post('/v1/account/sessions/anonymous') /** @var Utopia\Database\Database $dbForInternal */ /** @var MaxMind\Db\Reader $geodb */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ $protocol = $request->getProtocol(); @@ -618,7 +645,9 @@ App::post('/v1/account/sessions/anonymous') $limit = $project->getAttribute('auths', [])['limit'] ?? 0; if ($limit !== 0) { - $sum = $dbForInternal->count('users', [], APP_LIMIT_COUNT); + $sum = $dbForInternal->count('users', [ + new Query('deleted', Query::TYPE_EQUAL, [false]), + ], APP_LIMIT_COUNT); if ($sum >= $limit) { throw new Exception('Project registration is restricted. Contact your administrator for more information.', 501); @@ -644,6 +673,8 @@ App::post('/v1/account/sessions/anonymous') 'sessions' => [], 'tokens' => [], 'memberships' => [], + 'search' => $userId, + 'deleted' => false ])); Authorization::reset(); @@ -686,6 +717,11 @@ App::post('/v1/account/sessions/anonymous') ->setParam('resource', 'user/' . $user->getId()) ; + $usage + ->setParam('users.sessions.create', 1) + ->setParam('provider', 'anonymous') + ; + if (!Config::getParam('domainVerification')) { $response ->addHeader('X-Fallback-Cookies', \json_encode([Auth::$cookieName => Auth::encodeSession($user->getId(), $secret)])) @@ -771,10 +807,15 @@ App::get('/v1/account') ->label('sdk.response.model', Response::MODEL_USER) ->inject('response') ->inject('user') - ->action(function ($response, $user) { + ->inject('usage') + ->action(function ($response, $user, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Document $user */ + /** @var Appwrite\Stats\Stats $usage */ + $usage + ->setParam('users.read', 1) + ; $response->dynamic($user, Response::MODEL_USER); }); @@ -791,12 +832,17 @@ App::get('/v1/account/prefs') ->label('sdk.response.model', Response::MODEL_PREFERENCES) ->inject('response') ->inject('user') - ->action(function ($response, $user) { + ->inject('usage') + ->action(function ($response, $user, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Document $user */ + /** @var Appwrite\Stats\Stats $usage */ $prefs = $user->getAttribute('prefs', new \stdClass()); + $usage + ->setParam('users.read', 1) + ; $response->dynamic(new Document($prefs), Response::MODEL_PREFERENCES); }); @@ -814,10 +860,12 @@ App::get('/v1/account/sessions') ->inject('response') ->inject('user') ->inject('locale') - ->action(function ($response, $user, $locale) { + ->inject('usage') + ->action(function ($response, $user, $locale, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Document $user */ /** @var Utopia\Locale\Locale $locale */ + /** @var Appwrite\Stats\Stats $usage */ $sessions = $user->getAttribute('sessions', []); $current = Auth::sessionVerify($sessions, Auth::$secret); @@ -831,6 +879,9 @@ App::get('/v1/account/sessions') $sessions[$key] = $session; } + $usage + ->setParam('users.read', 1) + ; $response->dynamic(new Document([ 'sessions' => $sessions, 'sum' => count($sessions), @@ -853,13 +904,15 @@ App::get('/v1/account/logs') ->inject('locale') ->inject('geodb') ->inject('dbForInternal') - ->action(function ($response, $user, $locale, $geodb, $dbForInternal) { + ->inject('usage') + ->action(function ($response, $user, $locale, $geodb, $dbForInternal, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Document $project */ /** @var Utopia\Database\Document $user */ /** @var Utopia\Locale\Locale $locale */ /** @var MaxMind\Db\Reader $geodb */ /** @var Utopia\Database\Database $dbForInternal */ + /** @var Appwrite\Stats\Stats $usage */ $audit = new Audit($dbForInternal); @@ -906,6 +959,9 @@ App::get('/v1/account/logs') } + $usage + ->setParam('users.read', 1) + ; $response->dynamic(new Document(['logs' => $output]), Response::MODEL_LOG_LIST); }); @@ -925,11 +981,13 @@ App::get('/v1/account/sessions/:sessionId') ->inject('user') ->inject('locale') ->inject('dbForInternal') - ->action(function ($sessionId, $response, $user, $locale, $dbForInternal) { + ->inject('usage') + ->action(function ($sessionId, $response, $user, $locale, $dbForInternal, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Document $user */ /** @var Utopia\Locale\Locale $locale */ /** @var Utopia\Database\Database $dbForInternal */ + /** @var Appwrite\Stats\Stats $usage */ $sessions = $user->getAttribute('sessions', []); $sessionId = ($sessionId === 'current') @@ -948,6 +1006,10 @@ App::get('/v1/account/sessions/:sessionId') ->setAttribute('countryName', $countryName) ; + $usage + ->setParam('users.read', 1) + ; + return $response->dynamic($session, Response::MODEL_SESSION); } } @@ -972,13 +1034,18 @@ App::patch('/v1/account/name') ->inject('user') ->inject('dbForInternal') ->inject('audits') - ->action(function ($name, $response, $user, $dbForInternal, $audits) { + ->inject('usage') + ->action(function ($name, $response, $user, $dbForInternal, $audits, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Document $user */ /** @var Utopia\Database\Database $dbForInternal */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ - $user = $dbForInternal->updateDocument('users', $user->getId(), $user->setAttribute('name', $name)); + $user = $dbForInternal->updateDocument('users', $user->getId(), $user + ->setAttribute('name', $name) + ->setAttribute('search', implode(' ', [$user->getId(), $name, $user->getAttribute('email')])) + ); $audits ->setParam('userId', $user->getId()) @@ -986,6 +1053,10 @@ App::patch('/v1/account/name') ->setParam('resource', 'user/' . $user->getId()) ; + $usage + ->setParam('users.update', 1) + ; + $response->dynamic($user, Response::MODEL_USER); }); @@ -1007,11 +1078,13 @@ App::patch('/v1/account/password') ->inject('user') ->inject('dbForInternal') ->inject('audits') - ->action(function ($password, $oldPassword, $response, $user, $dbForInternal, $audits) { + ->inject('usage') + ->action(function ($password, $oldPassword, $response, $user, $dbForInternal, $audits, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Document $user */ /** @var Utopia\Database\Database $dbForInternal */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ // Check old password only if its an existing user. if ($user->getAttribute('passwordUpdate') !== 0 && !Auth::passwordVerify($oldPassword, $user->getAttribute('password'))) { // Double check user password @@ -1029,6 +1102,9 @@ App::patch('/v1/account/password') ->setParam('resource', 'user/' . $user->getId()) ; + $usage + ->setParam('users.update', 1) + ; $response->dynamic($user, Response::MODEL_USER); }); @@ -1050,11 +1126,13 @@ App::patch('/v1/account/email') ->inject('user') ->inject('dbForInternal') ->inject('audits') - ->action(function ($email, $password, $response, $user, $dbForInternal, $audits) { + ->inject('usage') + ->action(function ($email, $password, $response, $user, $dbForInternal, $audits, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Document $user */ /** @var Utopia\Database\Database $dbForInternal */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ $isAnonymousUser = is_null($user->getAttribute('email')) && is_null($user->getAttribute('password')); // Check if request is from an anonymous account for converting @@ -1066,11 +1144,18 @@ App::patch('/v1/account/email') } $email = \strtolower($email); + $profile = $dbForInternal->findOne('users', [new Query('email', Query::TYPE_EQUAL, [$email])]); // Get user by email address + + if ($profile) { + throw new Exception('User already registered', 409); + } + try { $user = $dbForInternal->updateDocument('users', $user->getId(), $user ->setAttribute('password', $isAnonymousUser ? Auth::passwordHash($password) : $user->getAttribute('password', '')) ->setAttribute('email', $email) ->setAttribute('emailVerification', false) // After this user needs to confirm mail again + ->setAttribute('search', implode(' ', [$user->getId(), $user->getAttribute('name'), $user->getAttribute('email')])) ); } catch(Duplicate $th) { throw new Exception('Email already exists', 409); @@ -1082,6 +1167,9 @@ App::patch('/v1/account/email') ->setParam('resource', 'user/' . $user->getId()) ; + $usage + ->setParam('users.update', 1) + ; $response->dynamic($user, Response::MODEL_USER); }); @@ -1102,11 +1190,13 @@ App::patch('/v1/account/prefs') ->inject('user') ->inject('dbForInternal') ->inject('audits') - ->action(function ($prefs, $response, $user, $dbForInternal, $audits) { + ->inject('usage') + ->action(function ($prefs, $response, $user, $dbForInternal, $audits, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Document $user */ /** @var Utopia\Database\Database $dbForInternal */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ $user = $dbForInternal->updateDocument('users', $user->getId(), $user->setAttribute('prefs', $prefs)); @@ -1115,6 +1205,9 @@ App::patch('/v1/account/prefs') ->setParam('resource', 'user/' . $user->getId()) ; + $usage + ->setParam('users.update', 1) + ; $response->dynamic($user, Response::MODEL_USER); }); @@ -1135,17 +1228,21 @@ App::delete('/v1/account') ->inject('dbForInternal') ->inject('audits') ->inject('events') - ->action(function ($request, $response, $user, $dbForInternal, $audits, $events) { + ->inject('usage') + ->action(function ($request, $response, $user, $dbForInternal, $audits, $events, $usage) { /** @var Utopia\Swoole\Request $request */ /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Document $user */ /** @var Utopia\Database\Database $dbForInternal */ /** @var Appwrite\Event\Event $audits */ /** @var Appwrite\Event\Event $events */ + /** @var Appwrite\Stats\Stats $usage */ $protocol = $request->getProtocol(); $user = $dbForInternal->updateDocument('users', $user->getId(), $user->setAttribute('status', false)); + // TODO Seems to be related to users.php/App::delete('/v1/users/:userId'). Can we share code between these two? Do todos below apply to users.php? + // TODO delete all tokens or only current session? // TODO delete all user data according to GDPR. Make sure everything is backed up and backups are deleted later /* @@ -1171,6 +1268,9 @@ App::delete('/v1/account') ; } + $usage + ->setParam('users.delete', 1) + ; $response ->addCookie(Auth::$cookieName . '_legacy', '', \time() - 3600, '/', Config::getParam('cookieDomain'), ('https' == $protocol), true, null) ->addCookie(Auth::$cookieName, '', \time() - 3600, '/', Config::getParam('cookieDomain'), ('https' == $protocol), true, Config::getParam('cookieSamesite')) @@ -1198,7 +1298,8 @@ App::delete('/v1/account/sessions/:sessionId') ->inject('locale') ->inject('audits') ->inject('events') - ->action(function ($sessionId, $request, $response, $user, $dbForInternal, $locale, $audits, $events) { + ->inject('usage') + ->action(function ($sessionId, $request, $response, $user, $dbForInternal, $locale, $audits, $events, $usage) { /** @var Utopia\Swoole\Request $request */ /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Document $user */ @@ -1206,6 +1307,7 @@ App::delete('/v1/account/sessions/:sessionId') /** @var Utopia\Locale\Locale $locale */ /** @var Appwrite\Event\Event $audits */ /** @var Appwrite\Event\Event $events */ + /** @var Appwrite\Stats\Stats $usage */ $protocol = $request->getProtocol(); $sessionId = ($sessionId === 'current') @@ -1252,6 +1354,10 @@ App::delete('/v1/account/sessions/:sessionId') ->setParam('eventData', $response->output($session, Response::MODEL_SESSION)) ; + $usage + ->setParam('users.sessions.delete', 1) + ->setParam('users.update', 1) + ; return $response->noContent(); } } @@ -1278,7 +1384,8 @@ App::delete('/v1/account/sessions') ->inject('locale') ->inject('audits') ->inject('events') - ->action(function ($request, $response, $user, $dbForInternal, $locale, $audits, $events) { + ->inject('usage') + ->action(function ($request, $response, $user, $dbForInternal, $locale, $audits, $events, $usage) { /** @var Utopia\Swoole\Request $request */ /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Document $user */ @@ -1286,6 +1393,7 @@ App::delete('/v1/account/sessions') /** @var Utopia\Locale\Locale $locale */ /** @var Appwrite\Event\Event $audits */ /** @var Appwrite\Event\Event $events */ + /** @var Appwrite\Stats\Stats $usage */ $protocol = $request->getProtocol(); $sessions = $user->getAttribute('sessions', []); @@ -1321,13 +1429,19 @@ App::delete('/v1/account/sessions') $dbForInternal->updateDocument('users', $user->getId(), $user->setAttribute('sessions', [])); + $numOfSessions = count($sessions); + $events ->setParam('eventData', $response->output(new Document([ 'sessions' => $sessions, - 'sum' => count($sessions), + 'sum' => $numOfSessions, ]), Response::MODEL_SESSION_LIST)) ; + $usage + ->setParam('users.sessions.delete', $numOfSessions) + ->setParam('users.update', 1) + ; $response->noContent(); }); @@ -1355,7 +1469,8 @@ App::post('/v1/account/recovery') ->inject('mails') ->inject('audits') ->inject('events') - ->action(function ($email, $url, $request, $response, $dbForInternal, $project, $locale, $mails, $audits, $events) { + ->inject('usage') + ->action(function ($email, $url, $request, $response, $dbForInternal, $project, $locale, $mails, $audits, $events, $usage) { /** @var Utopia\Swoole\Request $request */ /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ @@ -1364,12 +1479,13 @@ App::post('/v1/account/recovery') /** @var Appwrite\Event\Event $mails */ /** @var Appwrite\Event\Event $audits */ /** @var Appwrite\Event\Event $events */ + /** @var Appwrite\Stats\Stats $usage */ $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::$roles); $isAppUser = Auth::isAppUser(Authorization::$roles); $email = \strtolower($email); - $profile = $dbForInternal->findOne('users', [new Query('email', Query::TYPE_EQUAL, [$email])]); // Get user by email address + $profile = $dbForInternal->findOne('users', [new Query('deleted', Query::TYPE_EQUAL, [false]), new Query('email', Query::TYPE_EQUAL, [$email])]); // Get user by email address if (!$profile) { throw new Exception('User not found', 404); @@ -1431,6 +1547,9 @@ App::post('/v1/account/recovery') ->setParam('resource', 'user/' . $profile->getId()) ; + $usage + ->setParam('users.update', 1) + ; $response->setStatusCode(Response::STATUS_CODE_CREATED); $response->dynamic($recovery, Response::MODEL_TOKEN); }); @@ -1456,10 +1575,12 @@ App::put('/v1/account/recovery') ->inject('response') ->inject('dbForInternal') ->inject('audits') - ->action(function ($userId, $secret, $password, $passwordAgain, $response, $dbForInternal, $audits) { + ->inject('usage') + ->action(function ($userId, $secret, $password, $passwordAgain, $response, $dbForInternal, $audits, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ if ($password !== $passwordAgain) { throw new Exception('Passwords must match', 400); @@ -1467,7 +1588,7 @@ App::put('/v1/account/recovery') $profile = $dbForInternal->getDocument('users', $userId); - if ($profile->isEmpty()) { + if ($profile->isEmpty() || $profile->getAttribute('deleted')) { throw new Exception('User not found', 404); } @@ -1487,10 +1608,9 @@ App::put('/v1/account/recovery') ); /** - * We act like we're updating and validating - * the recovery token but actually we don't need it anymore. - */ - + * We act like we're updating and validating + * the recovery token but actually we don't need it anymore. + */ foreach ($tokens as $key => $token) { if ($recovery === $token->getId()) { $recovery = $token; @@ -1506,6 +1626,9 @@ App::put('/v1/account/recovery') ->setParam('resource', 'user/' . $profile->getId()) ; + $usage + ->setParam('users.update', 1) + ; $response->dynamic($recovery, Response::MODEL_TOKEN); }); @@ -1533,7 +1656,8 @@ App::post('/v1/account/verification') ->inject('audits') ->inject('events') ->inject('mails') - ->action(function ($url, $request, $response, $project, $user, $dbForInternal, $locale, $audits, $events, $mails) { + ->inject('usage') + ->action(function ($url, $request, $response, $project, $user, $dbForInternal, $locale, $audits, $events, $mails, $usage) { /** @var Utopia\Swoole\Request $request */ /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Document $project */ @@ -1543,6 +1667,7 @@ App::post('/v1/account/verification') /** @var Appwrite\Event\Event $audits */ /** @var Appwrite\Event\Event $events */ /** @var Appwrite\Event\Event $mails */ + /** @var Appwrite\Stats\Stats $usage */ $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::$roles); $isAppUser = Auth::isAppUser(Authorization::$roles); @@ -1600,6 +1725,9 @@ App::post('/v1/account/verification') ->setParam('resource', 'user/' . $user->getId()) ; + $usage + ->setParam('users.update', 1) + ; $response->setStatusCode(Response::STATUS_CODE_CREATED); $response->dynamic($verification, Response::MODEL_TOKEN); }); @@ -1624,11 +1752,13 @@ App::put('/v1/account/verification') ->inject('user') ->inject('dbForInternal') ->inject('audits') - ->action(function ($userId, $secret, $response, $user, $dbForInternal, $audits) { + ->inject('usage') + ->action(function ($userId, $secret, $response, $user, $dbForInternal, $audits, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Document $user */ /** @var Utopia\Database\Database $dbForInternal */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ $profile = $dbForInternal->getDocument('users', $userId); @@ -1648,9 +1778,9 @@ App::put('/v1/account/verification') $profile = $dbForInternal->updateDocument('users', $profile->getId(), $profile->setAttribute('emailVerification', true)); /** - * We act like we're updating and validating - * the verification token but actually we don't need it anymore. - */ + * We act like we're updating and validating + * the verification token but actually we don't need it anymore. + */ foreach ($tokens as $key => $token) { if ($token->getId() === $verification) { $verification = $token; @@ -1666,5 +1796,8 @@ App::put('/v1/account/verification') ->setParam('resource', 'user/' . $user->getId()) ; + $usage + ->setParam('users.update', 1) + ; $response->dynamic($verification, Response::MODEL_TOKEN); }); diff --git a/app/controllers/api/database.php b/app/controllers/api/database.php index 6c284e207..d26d960d4 100644 --- a/app/controllers/api/database.php +++ b/app/controllers/api/database.php @@ -15,6 +15,7 @@ use Utopia\Database\Database; use Utopia\Database\Document; use Utopia\Database\Query; use Utopia\Database\Adapter\MariaDB; +use Utopia\Database\Validator\Authorization; use Utopia\Database\Validator\Key; use Utopia\Database\Validator\Permissions; use Utopia\Database\Validator\QueryValidator; @@ -41,12 +42,12 @@ use DeviceDetector\DeviceDetector; * @param Utopia\Database\Database $dbForInternal * @param Appwrite\Event\Event $database * @param Appwrite\Event\Event $audits + * @param Appwrite\Stats\Stats $usage * * @return Document Newly created attribute document */ -function createAttribute($collectionId, $attribute, $response, $dbForInternal, $database, $audits): Document +function createAttribute($collectionId, $attribute, $response, $dbForInternal, $database, $audits, $usage): Document { - $attributeId = $attribute->getId(); $type = $attribute->getAttribute('type', ''); $size = $attribute->getAttribute('size', 0); @@ -76,7 +77,7 @@ function createAttribute($collectionId, $attribute, $response, $dbForInternal, $ } try { - $attribute = $dbForInternal->createDocument('attributes', new Document([ + $attribute = new Document([ '$id' => $collectionId.'_'.$attributeId, 'key' => $attributeId, 'collectionId' => $collectionId, @@ -89,13 +90,19 @@ function createAttribute($collectionId, $attribute, $response, $dbForInternal, $ 'array' => $array, 'format' => $format, 'formatOptions' => $formatOptions, - 'filters' => $filters, - ])); - } catch (DuplicateException $th) { + ]); + + $dbForInternal->checkAttribute($collection, $attribute); + $attribute = $dbForInternal->createDocument('attributes', $attribute); + } + catch (DuplicateException $exception) { throw new Exception('Attribute already exists', 409); } + catch (LimitException $exception) { + throw new Exception('Attribute limit exceeded', 400); + } - $dbForInternal->purgeDocument('collections', $collectionId); + $dbForInternal->deleteCachedDocument('collections', $collectionId); // Pass clone of $attribute object to workers // so we can later modify Document to fit response model @@ -107,6 +114,8 @@ function createAttribute($collectionId, $attribute, $response, $dbForInternal, $ ->setParam('document', $clone) ; + $usage->setParam('database.collections.update', 1); + $audits ->setParam('event', 'database.attributes.create') ->setParam('resource', 'collection/'.$collection->getId()) @@ -139,10 +148,12 @@ App::post('/v1/database/collections') ->inject('dbForInternal') ->inject('dbForExternal') ->inject('audits') - ->action(function ($collectionId, $name, $permission, $read, $write, $response, $dbForInternal, $dbForExternal, $audits) { + ->inject('usage') + ->action(function ($collectionId, $name, $permission, $read, $write, $response, $dbForInternal, $dbForExternal, $audits, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForExternal*/ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ $collectionId = $collectionId == 'unique()' ? $dbForExternal->getId() : $collectionId; @@ -169,6 +180,8 @@ App::post('/v1/database/collections') ->setParam('data', $collection->getArrayCopy()) ; + $usage->setParam('database.collections.create', 1); + $response->setStatusCode(Response::STATUS_CODE_CREATED); $response->dynamic($collection, Response::MODEL_COLLECTION); }); @@ -191,16 +204,11 @@ App::get('/v1/database/collections') ->param('orderType', 'ASC', new WhiteList(['ASC', 'DESC'], true), 'Order result by ASC or DESC order.', true) ->inject('response') ->inject('dbForInternal') - ->action(function ($search, $limit, $offset, $after, $orderType, $response, $dbForInternal) { + ->inject('usage') + ->action(function ($search, $limit, $offset, $after, $orderType, $response, $dbForInternal, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ - $queries = []; - - if (!empty($search)) { - $queries[] = new Query('name', Query::TYPE_SEARCH, [$search]); - } - if (!empty($after)) { $afterCollection = $dbForInternal->getDocument('collections', $after); @@ -209,6 +217,14 @@ App::get('/v1/database/collections') } } + $queries = []; + + if (!empty($search)) { + $queries[] = new Query('name', Query::TYPE_SEARCH, [$search]); + } + + $usage->setParam('database.collections.read', 1); + $response->dynamic(new Document([ 'collections' => $dbForInternal->find('collections', $queries, $limit, $offset, [], [$orderType], $afterCollection ?? null), 'sum' => $dbForInternal->count('collections', $queries, APP_LIMIT_COUNT), @@ -229,7 +245,8 @@ App::get('/v1/database/collections/:collectionId') ->param('collectionId', '', new UID(), 'Collection unique ID.') ->inject('response') ->inject('dbForInternal') - ->action(function ($collectionId, $response, $dbForInternal) { + ->inject('usage') + ->action(function ($collectionId, $response, $dbForInternal, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ @@ -239,9 +256,191 @@ App::get('/v1/database/collections/:collectionId') throw new Exception('Collection not found', 404); } + $usage->setParam('database.collections.read', 1); + $response->dynamic($collection, Response::MODEL_COLLECTION); }); +App::get('/v1/database/usage') + ->desc('Get usage stats for the database') + ->groups(['api', 'database']) + ->label('scope', 'collections.read') + ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) + ->label('sdk.namespace', 'database') + ->label('sdk.method', 'getUsage') + ->label('sdk.response.code', Response::STATUS_CODE_OK) + ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) + ->label('sdk.response.model', Response::MODEL_USAGE_DATABASE) + ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true) + ->inject('response') + ->inject('dbForInternal') + ->action(function ($range, $response, $dbForInternal) { + /** @var Appwrite\Utopia\Response $response */ + /** @var Utopia\Database\Database $dbForConsole */ + /** @var Utopia\Database\Database $dbForInternal */ + /** @var Utopia\Registry\Registry $register */ + + $usage = []; + if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { + $period = [ + '24h' => [ + 'period' => '30m', + 'limit' => 48, + ], + '7d' => [ + 'period' => '1d', + 'limit' => 7, + ], + '30d' => [ + 'period' => '1d', + 'limit' => 30, + ], + '90d' => [ + 'period' => '1d', + 'limit' => 90, + ], + ]; + + $metrics = [ + 'database.documents.count', + 'database.collections.count', + 'database.collections.create', + 'database.collections.read', + 'database.collections.update', + 'database.collections.delete', + 'database.documents.create', + 'database.documents.read', + 'database.documents.update', + 'database.documents.delete' + ]; + + $stats = []; + + Authorization::skip(function() use ($dbForInternal, $period, $range, $metrics, &$stats) { + foreach ($metrics as $metric) { + $requestDocs = $dbForInternal->find('stats', [ + new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]), + new Query('metric', Query::TYPE_EQUAL, [$metric]), + ], $period[$range]['limit'], 0, ['time'], [Database::ORDER_DESC]); + + $stats[$metric] = []; + foreach ($requestDocs as $requestDoc) { + $stats[$metric][] = [ + 'value' => $requestDoc->getAttribute('value'), + 'date' => $requestDoc->getAttribute('time'), + ]; + } + $stats[$metric] = array_reverse($stats[$metric]); + } + }); + + $usage = new Document([ + 'range' => $range, + 'documents.count' => $stats["database.documents.count"], + 'collections.count' => $stats["database.collections.count"], + 'documents.create' => $stats["database.documents.create"], + 'documents.read' => $stats["database.documents.read"], + 'documents.update' => $stats["database.documents.update"], + 'documents.delete' => $stats["database.documents.delete"], + 'collections.create' => $stats["database.collections.create"], + 'collections.read' => $stats["database.collections.read"], + 'collections.update' => $stats["database.collections.update"], + 'collections.delete' => $stats["database.collections.delete"], + ]); + } + + $response->dynamic($usage, Response::MODEL_USAGE_DATABASE); + }); + +App::get('/v1/database/:collectionId/usage') + ->desc('Get usage stats for a collection') + ->groups(['api', 'database']) + ->label('scope', 'collections.read') + ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) + ->label('sdk.namespace', 'database') + ->label('sdk.method', 'getCollectionUsage') + ->label('sdk.response.code', Response::STATUS_CODE_OK) + ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) + ->label('sdk.response.model', Response::MODEL_USAGE_COLLECTION) + ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true) + ->param('collectionId', '', new UID(), 'Collection unique ID.') + ->inject('response') + ->inject('dbForInternal') + ->inject('dbForExternal') + ->action(function ($range, $collectionId, $response, $dbForInternal, $dbForExternal) { + /** @var Appwrite\Utopia\Response $response */ + /** @var Utopia\Database\Database $dbForConsole */ + /** @var Utopia\Database\Database $dbForInternal */ + /** @var Utopia\Registry\Registry $register */ + + $collection = $dbForExternal->getCollection($collectionId); + + if ($collection->isEmpty()) { + throw new Exception('Collection not found', 404); + } + + $usage = []; + if(App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { + $period = [ + '24h' => [ + 'period' => '30m', + 'limit' => 48, + ], + '7d' => [ + 'period' => '1d', + 'limit' => 7, + ], + '30d' => [ + 'period' => '1d', + 'limit' => 30, + ], + '90d' => [ + 'period' => '1d', + 'limit' => 90, + ], + ]; + + $metrics = [ + "database.collections.$collectionId.documents.count", + "database.collections.$collectionId.documents.create", + "database.collections.$collectionId.documents.read", + "database.collections.$collectionId.documents.update", + "database.collections.$collectionId.documents.delete", + ]; + + $stats = []; + + Authorization::skip(function() use ($dbForInternal, $period, $range, $metrics, &$stats) { + foreach ($metrics as $metric) { + $requestDocs = $dbForInternal->find('stats', [ + new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]), + new Query('metric', Query::TYPE_EQUAL, [$metric]), + ], $period[$range]['limit'], 0, ['time'], [Database::ORDER_DESC]); + + $stats[$metric] = []; + foreach ($requestDocs as $requestDoc) { + $stats[$metric][] = [ + 'value' => $requestDoc->getAttribute('value'), + 'date' => $requestDoc->getAttribute('time'), + ]; + } + $stats[$metric] = array_reverse($stats[$metric]); + } + }); + + $usage = new Document([ + 'range' => $range, + 'documents.count' => $stats["database.collections.$collectionId.documents.count"], + 'documents.create' => $stats["database.collections.$collectionId.documents.create"], + 'documents.read' => $stats["database.collections.$collectionId.documents.read"], + 'documents.update' => $stats["database.collections.$collectionId.documents.update"], + 'documents.delete' => $stats["database.collections.$collectionId.documents.delete"] + ]); + } + + $response->dynamic($usage, Response::MODEL_USAGE_COLLECTION); + }); + App::get('/v1/database/collections/:collectionId/logs') ->desc('List Collection Logs') ->groups(['api', 'database']) @@ -358,10 +557,12 @@ App::put('/v1/database/collections/:collectionId') ->inject('response') ->inject('dbForInternal') ->inject('audits') - ->action(function ($collectionId, $name, $permission, $read, $write, $response, $dbForInternal, $audits) { + ->inject('usage') + ->action(function ($collectionId, $name, $permission, $read, $write, $response, $dbForInternal, $audits, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ $collection = $dbForInternal->getDocument('collections', $collectionId); @@ -389,6 +590,8 @@ App::put('/v1/database/collections/:collectionId') throw new Exception('Bad structure. '.$exception->getMessage(), 400); } + $usage->setParam('database.collections.update', 1); + $audits ->setParam('event', 'database.collections.update') ->setParam('resource', 'collection/'.$collection->getId()) @@ -416,12 +619,14 @@ App::delete('/v1/database/collections/:collectionId') ->inject('events') ->inject('audits') ->inject('deletes') - ->action(function ($collectionId, $response, $dbForInternal, $dbForExternal, $events, $audits, $deletes) { + ->inject('usage') + ->action(function ($collectionId, $response, $dbForInternal, $dbForExternal, $events, $audits, $deletes, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ /** @var Utopia\Database\Database $dbForExternal */ /** @var Appwrite\Event\Event $events */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $audits */ $collection = $dbForInternal->getDocument('collections', $collectionId); @@ -429,12 +634,17 @@ App::delete('/v1/database/collections/:collectionId') throw new Exception('Collection not found', 404); } - $dbForExternal->deleteCollection($collectionId); // TDOD move to DB worker - if (!$dbForInternal->deleteDocument('collections', $collectionId)) { throw new Exception('Failed to remove collection from DB', 500); } + $deletes + ->setParam('type', DELETE_TYPE_DOCUMENT) + ->setParam('document', $collection) + ; + + $usage->setParam('database.collections.delete', 1); + $events ->setParam('eventData', $response->output($collection, Response::MODEL_COLLECTION)) ; @@ -459,7 +669,7 @@ App::post('/v1/database/collections/:collectionId/attributes/string') ->label('sdk.description', '/docs/references/database/create-attribute-string.md') ->label('sdk.response.code', Response::STATUS_CODE_CREATED) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) - ->label('sdk.response.model', Response::MODEL_ATTRIBUTE) + ->label('sdk.response.model', Response::MODEL_ATTRIBUTE_STRING) ->param('collectionId', '', new UID(), 'Collection unique ID. You can create a new collection using the Database service [server integration](/docs/server/database#createCollection).') ->param('attributeId', '', new Key(), 'Attribute ID.') ->param('size', null, new Integer(), 'Attribute size for text attributes, in number of characters.') @@ -470,12 +680,13 @@ App::post('/v1/database/collections/:collectionId/attributes/string') ->inject('dbForInternal') ->inject('database') ->inject('audits') - ->action(function ($collectionId, $attributeId, $size, $required, $default, $array, $response, $dbForInternal, $database, $audits) { + ->inject('usage') + ->action(function ($collectionId, $attributeId, $size, $required, $default, $array, $response, $dbForInternal, $database, $audits, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal*/ - /** @var Utopia\Database\Database $dbForExternal*/ /** @var Appwrite\Event\Event $database */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ // Ensure attribute default is within required size $validator = new Text($size); @@ -490,7 +701,7 @@ App::post('/v1/database/collections/:collectionId/attributes/string') 'required' => $required, 'default' => $default, 'array' => $array, - ]), $response, $dbForInternal, $database, $audits); + ]), $response, $dbForInternal, $database, $audits, $usage); $response->dynamic($attribute, Response::MODEL_ATTRIBUTE_STRING); }); @@ -506,7 +717,7 @@ App::post('/v1/database/collections/:collectionId/attributes/email') ->label('sdk.description', '/docs/references/database/create-attribute-email.md') ->label('sdk.response.code', Response::STATUS_CODE_CREATED) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) - ->label('sdk.response.model', Response::MODEL_ATTRIBUTE) + ->label('sdk.response.model', Response::MODEL_ATTRIBUTE_EMAIL) ->param('collectionId', '', new UID(), 'Collection unique ID. You can create a new collection using the Database service [server integration](/docs/server/database#createCollection).') ->param('attributeId', '', new Key(), 'Attribute ID.') ->param('required', null, new Boolean(), 'Is attribute required?') @@ -516,11 +727,13 @@ App::post('/v1/database/collections/:collectionId/attributes/email') ->inject('dbForInternal') ->inject('database') ->inject('audits') - ->action(function ($collectionId, $attributeId, $required, $default, $array, $response, $dbForInternal, $database, $audits) { + ->inject('usage') + ->action(function ($collectionId, $attributeId, $required, $default, $array, $response, $dbForInternal, $database, $audits, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal*/ /** @var Appwrite\Event\Event $database */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ $attribute = createAttribute($collectionId, new Document([ '$id' => $attributeId, @@ -530,7 +743,7 @@ App::post('/v1/database/collections/:collectionId/attributes/email') 'default' => $default, 'array' => $array, 'format' => APP_DATABASE_ATTRIBUTE_EMAIL, - ]), $response, $dbForInternal, $database, $audits); + ]), $response, $dbForInternal, $database, $audits, $usage); $response->dynamic($attribute, Response::MODEL_ATTRIBUTE_EMAIL); }); @@ -595,7 +808,7 @@ App::post('/v1/database/collections/:collectionId/attributes/ip') ->label('sdk.description', '/docs/references/database/create-attribute-ip.md') ->label('sdk.response.code', Response::STATUS_CODE_CREATED) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) - ->label('sdk.response.model', Response::MODEL_ATTRIBUTE) + ->label('sdk.response.model', Response::MODEL_ATTRIBUTE_IP) ->param('collectionId', '', new UID(), 'Collection unique ID. You can create a new collection using the Database service [server integration](/docs/server/database#createCollection).') ->param('attributeId', '', new Key(), 'Attribute ID.') ->param('required', null, new Boolean(), 'Is attribute required?') @@ -605,11 +818,13 @@ App::post('/v1/database/collections/:collectionId/attributes/ip') ->inject('dbForInternal') ->inject('database') ->inject('audits') - ->action(function ($collectionId, $attributeId, $required, $default, $array, $response, $dbForInternal, $database, $audits) { + ->inject('usage') + ->action(function ($collectionId, $attributeId, $required, $default, $array, $response, $dbForInternal, $database, $audits, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal*/ /** @var Appwrite\Event\Event $database */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ $attribute = createAttribute($collectionId, new Document([ '$id' => $attributeId, @@ -619,13 +834,13 @@ App::post('/v1/database/collections/:collectionId/attributes/ip') 'default' => $default, 'array' => $array, 'format' => APP_DATABASE_ATTRIBUTE_IP, - ]), $response, $dbForInternal, $database, $audits); + ]), $response, $dbForInternal, $database, $audits, $usage); $response->dynamic($attribute, Response::MODEL_ATTRIBUTE_IP); }); App::post('/v1/database/collections/:collectionId/attributes/url') - ->desc('Create IP Address Attribute') + ->desc('Create URL Attribute') ->groups(['api', 'database']) ->label('event', 'database.attributes.create') ->label('scope', 'collections.write') @@ -635,7 +850,7 @@ App::post('/v1/database/collections/:collectionId/attributes/url') ->label('sdk.description', '/docs/references/database/create-attribute-url.md') ->label('sdk.response.code', Response::STATUS_CODE_CREATED) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) - ->label('sdk.response.model', Response::MODEL_ATTRIBUTE) + ->label('sdk.response.model', Response::MODEL_ATTRIBUTE_URL) ->param('collectionId', '', new UID(), 'Collection unique ID. You can create a new collection using the Database service [server integration](/docs/server/database#createCollection).') ->param('attributeId', '', new Key(), 'Attribute ID.') ->param('required', null, new Boolean(), 'Is attribute required?') @@ -645,11 +860,13 @@ App::post('/v1/database/collections/:collectionId/attributes/url') ->inject('dbForInternal') ->inject('database') ->inject('audits') - ->action(function ($collectionId, $attributeId, $required, $default, $array, $response, $dbForInternal, $database, $audits) { + ->inject('usage') + ->action(function ($collectionId, $attributeId, $required, $default, $array, $response, $dbForInternal, $database, $audits, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForExternal*/ /** @var Appwrite\Event\Event $database */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ $attribute = createAttribute($collectionId, new Document([ '$id' => $attributeId, @@ -659,7 +876,7 @@ App::post('/v1/database/collections/:collectionId/attributes/url') 'default' => $default, 'array' => $array, 'format' => APP_DATABASE_ATTRIBUTE_URL, - ]), $response, $dbForInternal, $database, $audits); + ]), $response, $dbForInternal, $database, $audits, $usage); $response->dynamic($attribute, Response::MODEL_ATTRIBUTE_URL); }); @@ -675,7 +892,7 @@ App::post('/v1/database/collections/:collectionId/attributes/integer') ->label('sdk.description', '/docs/references/database/create-attribute-integer.md') ->label('sdk.response.code', Response::STATUS_CODE_CREATED) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) - ->label('sdk.response.model', Response::MODEL_ATTRIBUTE) + ->label('sdk.response.model', Response::MODEL_ATTRIBUTE_INTEGER) ->param('collectionId', '', new UID(), 'Collection unique ID. You can create a new collection using the Database service [server integration](/docs/server/database#createCollection).') ->param('attributeId', '', new Key(), 'Attribute ID.') ->param('required', null, new Boolean(), 'Is attribute required?') @@ -687,11 +904,13 @@ App::post('/v1/database/collections/:collectionId/attributes/integer') ->inject('dbForInternal') ->inject('database') ->inject('audits') - ->action(function ($collectionId, $attributeId, $required, $min, $max, $default, $array, $response, $dbForInternal, $database, $audits) { + ->inject('usage') + ->action(function ($collectionId, $attributeId, $required, $min, $max, $default, $array, $response, $dbForInternal, $database, $audits, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal*/ /** @var Appwrite\Event\Event $database */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ // Ensure attribute default is within range $min = (is_null($min)) ? PHP_INT_MIN : \intval($min); @@ -714,7 +933,7 @@ App::post('/v1/database/collections/:collectionId/attributes/integer') 'min' => $min, 'max' => $max, ], - ]), $response, $dbForInternal, $database, $audits); + ]), $response, $dbForInternal, $database, $audits, $usage); $formatOptions = $attribute->getAttribute('formatOptions', []); @@ -737,7 +956,7 @@ App::post('/v1/database/collections/:collectionId/attributes/float') ->label('sdk.description', '/docs/references/database/create-attribute-float.md') ->label('sdk.response.code', Response::STATUS_CODE_CREATED) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) - ->label('sdk.response.model', Response::MODEL_ATTRIBUTE) + ->label('sdk.response.model', Response::MODEL_ATTRIBUTE_FLOAT) ->param('collectionId', '', new UID(), 'Collection unique ID. You can create a new collection using the Database service [server integration](/docs/server/database#createCollection).') ->param('attributeId', '', new Key(), 'Attribute ID.') ->param('required', null, new Boolean(), 'Is attribute required?') @@ -749,11 +968,13 @@ App::post('/v1/database/collections/:collectionId/attributes/float') ->inject('dbForInternal') ->inject('database') ->inject('audits') - ->action(function ($collectionId, $attributeId, $required, $min, $max, $default, $array, $response, $dbForInternal, $database, $audits) { + ->inject('usage') + ->action(function ($collectionId, $attributeId, $required, $min, $max, $default, $array, $response, $dbForInternal, $database, $audits, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal*/ /** @var Appwrite\Event\Event $database */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ // Ensure attribute default is within range $min = (is_null($min)) ? PHP_FLOAT_MIN : \floatval($min); @@ -776,7 +997,7 @@ App::post('/v1/database/collections/:collectionId/attributes/float') 'min' => $min, 'max' => $max, ], - ]), $response, $dbForInternal, $database, $audits); + ]), $response, $dbForInternal, $database, $audits, $usage); $formatOptions = $attribute->getAttribute('formatOptions', []); @@ -799,7 +1020,7 @@ App::post('/v1/database/collections/:collectionId/attributes/boolean') ->label('sdk.description', '/docs/references/database/create-attribute-boolean.md') ->label('sdk.response.code', Response::STATUS_CODE_CREATED) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) - ->label('sdk.response.model', Response::MODEL_ATTRIBUTE) + ->label('sdk.response.model', Response::MODEL_ATTRIBUTE_BOOLEAN) ->param('collectionId', '', new UID(), 'Collection unique ID. You can create a new collection using the Database service [server integration](/docs/server/database#createCollection).') ->param('attributeId', '', new Key(), 'Attribute ID.') ->param('required', null, new Boolean(), 'Is attribute required?') @@ -809,11 +1030,13 @@ App::post('/v1/database/collections/:collectionId/attributes/boolean') ->inject('dbForInternal') ->inject('database') ->inject('audits') - ->action(function ($collectionId, $attributeId, $required, $default, $array, $response, $dbForInternal, $database, $audits) { + ->inject('usage') + ->action(function ($collectionId, $attributeId, $required, $default, $array, $response, $dbForInternal, $database, $audits, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal*/ /** @var Appwrite\Event\Event $database */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ $attribute = createAttribute($collectionId, new Document([ '$id' => $attributeId, @@ -822,7 +1045,7 @@ App::post('/v1/database/collections/:collectionId/attributes/boolean') 'required' => $required, 'default' => $default, 'array' => $array, - ]), $response, $dbForInternal, $database, $audits); + ]), $response, $dbForInternal, $database, $audits, $usage); $response->dynamic($attribute, Response::MODEL_ATTRIBUTE_BOOLEAN); }); @@ -841,7 +1064,8 @@ App::get('/v1/database/collections/:collectionId/attributes') ->param('collectionId', '', new UID(), 'Collection unique ID. You can create a new collection using the Database service [server integration](/docs/server/database#createCollection).') ->inject('response') ->inject('dbForInternal') - ->action(function ($collectionId, $response, $dbForInternal) { + ->inject('usage') + ->action(function ($collectionId, $response, $dbForInternal, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ @@ -853,6 +1077,8 @@ App::get('/v1/database/collections/:collectionId/attributes') $attributes = $collection->getAttribute('attributes'); + $usage->setParam('database.collections.read', 1); + $response->dynamic(new Document([ 'sum' => \count($attributes), 'attributes' => $attributes @@ -869,12 +1095,20 @@ App::get('/v1/database/collections/:collectionId/attributes/:attributeId') ->label('sdk.description', '/docs/references/database/get-attribute.md') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) - ->label('sdk.response.model', Response::MODEL_ATTRIBUTE) + ->label('sdk.response.model', [ + Response::MODEL_ATTRIBUTE_BOOLEAN, + Response::MODEL_ATTRIBUTE_INTEGER, + Response::MODEL_ATTRIBUTE_FLOAT, + Response::MODEL_ATTRIBUTE_EMAIL, + Response::MODEL_ATTRIBUTE_URL, + Response::MODEL_ATTRIBUTE_IP, + Response::MODEL_ATTRIBUTE_STRING,])// needs to be last, since its condition would dominate any other string attribute ->param('collectionId', '', new UID(), 'Collection unique ID. You can create a new collection using the Database service [server integration](/docs/server/database#createCollection).') ->param('attributeId', '', new Key(), 'Attribute ID.') ->inject('response') ->inject('dbForInternal') - ->action(function ($collectionId, $attributeId, $response, $dbForInternal) { + ->inject('usage') + ->action(function ($collectionId, $attributeId, $response, $dbForInternal, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ @@ -908,6 +1142,8 @@ App::get('/v1/database/collections/:collectionId/attributes/:attributeId') default => Response::MODEL_ATTRIBUTE, }; + $usage->setParam('database.collections.read', 1); + $response->dynamic($attribute, $model); }); @@ -929,12 +1165,14 @@ App::delete('/v1/database/collections/:collectionId/attributes/:attributeId') ->inject('database') ->inject('events') ->inject('audits') - ->action(function ($collectionId, $attributeId, $response, $dbForInternal, $database, $events, $audits) { + ->inject('usage') + ->action(function ($collectionId, $attributeId, $response, $dbForInternal, $database, $events, $audits, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ /** @var Appwrite\Event\Event $database */ /** @var Appwrite\Event\Event $events */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ $collection = $dbForInternal->getDocument('collections', $collectionId); @@ -949,7 +1187,7 @@ App::delete('/v1/database/collections/:collectionId/attributes/:attributeId') } $attribute = $dbForInternal->updateDocument('attributes', $attribute->getId(), $attribute->setAttribute('status', 'deleting')); - $dbForInternal->purgeDocument('collections', $collectionId); + $dbForInternal->deleteCachedDocument('collections', $collectionId); $database ->setParam('type', DATABASE_TYPE_DELETE_ATTRIBUTE) @@ -957,6 +1195,8 @@ App::delete('/v1/database/collections/:collectionId/attributes/:attributeId') ->setParam('document', $attribute) ; + $usage->setParam('database.collections.update', 1); + $events ->setParam('payload', $response->output($attribute, Response::MODEL_ATTRIBUTE)) ; @@ -991,11 +1231,13 @@ App::post('/v1/database/collections/:collectionId/indexes') ->inject('dbForInternal') ->inject('database') ->inject('audits') - ->action(function ($collectionId, $indexId, $type, $attributes, $orders, $response, $dbForInternal, $database, $audits) { + ->inject('usage') + ->action(function ($collectionId, $indexId, $type, $attributes, $orders, $response, $dbForInternal, $database, $audits, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ /** @var Appwrite\Event\Event $database */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $audits */ $collection = $dbForInternal->getDocument('collections', $collectionId); @@ -1052,7 +1294,7 @@ App::post('/v1/database/collections/:collectionId/indexes') throw new Exception('Index already exists', 409); } - $dbForInternal->purgeDocument('collections', $collectionId); + $dbForInternal->deleteCachedDocument('collections', $collectionId); $database ->setParam('type', DATABASE_TYPE_CREATE_INDEX) @@ -1060,6 +1302,8 @@ App::post('/v1/database/collections/:collectionId/indexes') ->setParam('document', $index) ; + $usage->setParam('database.collections.update', 1); + $audits ->setParam('event', 'database.indexes.create') ->setParam('resource', 'collection/'.$collection->getId()) @@ -1084,7 +1328,8 @@ App::get('/v1/database/collections/:collectionId/indexes') ->param('collectionId', '', new UID(), 'Collection unique ID. You can create a new collection using the Database service [server integration](/docs/server/database#createCollection).') ->inject('response') ->inject('dbForInternal') - ->action(function ($collectionId, $response, $dbForInternal) { + ->inject('usage') + ->action(function ($collectionId, $response, $dbForInternal, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ @@ -1102,6 +1347,8 @@ App::get('/v1/database/collections/:collectionId/indexes') ])]); }, $indexes); + $usage->setParam('database.collections.read', 1); + $response->dynamic(new Document([ 'sum' => \count($indexes), 'attributes' => $indexes, @@ -1123,7 +1370,8 @@ App::get('/v1/database/collections/:collectionId/indexes/:indexId') ->param('indexId', null, new Key(), 'Index ID.') ->inject('response') ->inject('dbForInternal') - ->action(function ($collectionId, $indexId, $response, $dbForInternal) { + ->inject('usage') + ->action(function ($collectionId, $indexId, $response, $dbForInternal, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ @@ -1145,6 +1393,8 @@ App::get('/v1/database/collections/:collectionId/indexes/:indexId') $index = new Document([\array_merge($indexes[$indexIndex], [ 'collectionId' => $collectionId, ])]); + + $usage->setParam('database.collections.read', 1); $response->dynamic($index, Response::MODEL_INDEX); }); @@ -1167,12 +1417,14 @@ App::delete('/v1/database/collections/:collectionId/indexes/:indexId') ->inject('database') ->inject('events') ->inject('audits') - ->action(function ($collectionId, $indexId, $response, $dbForInternal, $database, $events, $audits) { + ->inject('usage') + ->action(function ($collectionId, $indexId, $response, $dbForInternal, $database, $events, $audits, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ /** @var Appwrite\Event\Event $database */ /** @var Appwrite\Event\Event $events */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ $collection = $dbForInternal->getDocument('collections', $collectionId); @@ -1187,7 +1439,7 @@ App::delete('/v1/database/collections/:collectionId/indexes/:indexId') } $index = $dbForInternal->updateDocument('indexes', $index->getId(), $index->setAttribute('status', 'deleting')); - $dbForInternal->purgeDocument('collections', $collectionId); + $dbForInternal->deleteCachedDocument('collections', $collectionId); $database ->setParam('type', DATABASE_TYPE_DELETE_INDEX) @@ -1195,6 +1447,8 @@ App::delete('/v1/database/collections/:collectionId/indexes/:indexId') ->setParam('document', $index) ; + $usage->setParam('database.collections.update', 1); + $events ->setParam('payload', $response->output($index, Response::MODEL_INDEX)) ; @@ -1230,12 +1484,14 @@ App::post('/v1/database/collections/:collectionId/documents') ->inject('dbForExternal') ->inject('user') ->inject('audits') - ->action(function ($documentId, $collectionId, $data, $read, $write, $response, $dbForInternal, $dbForExternal, $user, $audits) { + ->inject('usage') + ->action(function ($documentId, $collectionId, $data, $read, $write, $response, $dbForInternal, $dbForExternal, $user, $audits, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ /** @var Utopia\Database\Database $dbForExternal */ /** @var Utopia\Database\Document $user */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ $data = (\is_string($data)) ? \json_decode($data, true) : $data; // Cast to JSON array @@ -1253,13 +1509,28 @@ App::post('/v1/database/collections/:collectionId/documents') throw new Exception('Collection not found', 404); } + // Check collection permissions when enforced + if ($collection->getAttribute('permission') === 'collection') { + $validator = new Authorization('write'); + if (!$validator->isValid($collection->getWrite())) { + throw new Exception('Unauthorized permissions', 401); + } + } + $data['$collection'] = $collection->getId(); // Adding this param to make API easier for developers $data['$id'] = $documentId == 'unique()' ? $dbForExternal->getId() : $documentId; $data['$read'] = (is_null($read) && !$user->isEmpty()) ? ['user:'.$user->getId()] : $read ?? []; // By default set read permissions for user $data['$write'] = (is_null($write) && !$user->isEmpty()) ? ['user:'.$user->getId()] : $write ?? []; // By default set write permissions for user try { - $document = $dbForExternal->createDocument($collectionId, new Document($data)); + if ($collection->getAttribute('permission') === 'collection') { + /** @var Document $document */ + $document = Authorization::skip(function() use ($dbForExternal, $collectionId, $data) { + return $dbForExternal->createDocument($collectionId, new Document($data)); + }); + } else { + $document = $dbForExternal->createDocument($collectionId, new Document($data)); + } } catch (StructureException $exception) { throw new Exception($exception->getMessage(), 400); @@ -1268,6 +1539,11 @@ App::post('/v1/database/collections/:collectionId/documents') throw new Exception('Document already exists', 409); } + $usage + ->setParam('database.documents.create', 1) + ->setParam('collectionId', $collectionId) + ; + $audits ->setParam('event', 'database.documents.create') ->setParam('resource', 'document/'.$document->getId()) @@ -1299,10 +1575,12 @@ App::get('/v1/database/collections/:collectionId/documents') ->inject('response') ->inject('dbForInternal') ->inject('dbForExternal') - ->action(function ($collectionId, $queries, $limit, $offset, $after, $orderAttributes, $orderTypes, $response, $dbForInternal, $dbForExternal) { + ->inject('usage') + ->action(function ($collectionId, $queries, $limit, $offset, $after, $orderAttributes, $orderTypes, $response, $dbForInternal, $dbForExternal, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ /** @var Utopia\Database\Database $dbForExternal */ + /** @var Appwrite\Stats\Stats $usage */ $collection = $dbForInternal->getDocument('collections', $collectionId); @@ -1310,6 +1588,14 @@ App::get('/v1/database/collections/:collectionId/documents') throw new Exception('Collection not found', 404); } + // Check collection permissions when enforced + if ($collection->getAttribute('permission') === 'collection') { + $validator = new Authorization('read'); + if (!$validator->isValid($collection->getRead())) { + throw new Exception('Unauthorized permissions', 401); + } + } + $queries = \array_map(function ($query) { return Query::parse($query); }, $queries); @@ -1329,9 +1615,23 @@ App::get('/v1/database/collections/:collectionId/documents') } } + if ($collection->getAttribute('permission') === 'collection') { + /** @var Document[] $documents */ + $documents = Authorization::skip(function() use ($dbForExternal, $collectionId, $queries, $limit, $offset, $orderAttributes, $orderTypes, $afterDocument) { + return $dbForExternal->find($collectionId, $queries, $limit, $offset, $orderAttributes, $orderTypes, $afterDocument ?? null); + }); + } else { + $documents = $dbForExternal->find($collectionId, $queries, $limit, $offset, $orderAttributes, $orderTypes, $afterDocument ?? null); + } + + $usage + ->setParam('database.documents.read', 1) + ->setParam('collectionId', $collectionId) + ; + $response->dynamic(new Document([ 'sum' => $dbForExternal->count($collectionId, $queries, APP_LIMIT_COUNT), - 'documents' => $dbForExternal->find($collectionId, $queries, $limit, $offset, $orderAttributes, $orderTypes, $afterDocument ?? null), + 'documents' => $documents, ]), Response::MODEL_DOCUMENT_LIST); }); @@ -1351,7 +1651,8 @@ App::get('/v1/database/collections/:collectionId/documents/:documentId') ->inject('response') ->inject('dbForInternal') ->inject('dbForExternal') - ->action(function ($collectionId, $documentId, $response, $dbForInternal, $dbForExternal) { + ->inject('usage') + ->action(function ($collectionId, $documentId, $response, $dbForInternal, $dbForExternal, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $$dbForInternal */ /** @var Utopia\Database\Database $dbForExternal */ @@ -1362,12 +1663,32 @@ App::get('/v1/database/collections/:collectionId/documents/:documentId') throw new Exception('Collection not found', 404); } - $document = $dbForExternal->getDocument($collectionId, $documentId); + // Check collection permissions when enforced + if ($collection->getAttribute('permission') === 'collection') { + $validator = new Authorization('read'); + if (!$validator->isValid($collection->getRead())) { + throw new Exception('Unauthorized permissions', 401); + } + } + + if ($collection->getAttribute('permission') === 'collection') { + /** @var Document $document */ + $document = Authorization::skip(function() use ($dbForExternal, $collectionId, $documentId) { + return $dbForExternal->getDocument($collectionId, $documentId); + }); + } else { + $document = $dbForExternal->getDocument($collectionId, $documentId); + } if ($document->isEmpty()) { throw new Exception('No document found', 404); } + $usage + ->setParam('database.documents.read', 1) + ->setParam('collectionId', $collectionId) + ; + $response->dynamic($document, Response::MODEL_DOCUMENT); }); @@ -1392,11 +1713,13 @@ App::patch('/v1/database/collections/:collectionId/documents/:documentId') ->inject('dbForInternal') ->inject('dbForExternal') ->inject('audits') - ->action(function ($collectionId, $documentId, $data, $read, $write, $response, $dbForInternal, $dbForExternal, $audits) { + ->inject('usage') + ->action(function ($collectionId, $documentId, $data, $read, $write, $response, $dbForInternal, $dbForExternal, $audits, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ /** @var Utopia\Database\Database $dbForExternal */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ $collection = $dbForInternal->getDocument('collections', $collectionId); @@ -1404,6 +1727,14 @@ App::patch('/v1/database/collections/:collectionId/documents/:documentId') throw new Exception('Collection not found', 404); } + // Check collection permissions when enforced + if ($collection->getAttribute('permission') === 'collection') { + $validator = new Authorization('write'); + if (!$validator->isValid($collection->getWrite())) { + throw new Exception('Unauthorized permissions', 401); + } + } + $document = $dbForExternal->getDocument($collectionId, $documentId); if ($document->isEmpty()) { @@ -1428,7 +1759,14 @@ App::patch('/v1/database/collections/:collectionId/documents/:documentId') $data['$write'] = (is_null($write)) ? ($document->getWrite() ?? []) : $write; // By default inherit write permissions try { - $document = $dbForExternal->updateDocument($collection->getId(), $document->getId(), new Document($data)); + if ($collection->getAttribute('permission') === 'collection') { + /** @var Document $document */ + $document = Authorization::skip(function() use ($dbForExternal, $collection, $document, $data) { + return $dbForExternal->updateDocument($collection->getId(), $document->getId(), new Document($data)); + }); + } else { + $document = $dbForExternal->updateDocument($collection->getId(), $document->getId(), new Document($data)); + } } catch (AuthorizationException $exception) { throw new Exception('Unauthorized permissions', 401); @@ -1439,6 +1777,11 @@ App::patch('/v1/database/collections/:collectionId/documents/:documentId') catch (StructureException $exception) { throw new Exception($exception->getMessage(), 400); } + + $usage + ->setParam('database.documents.update', 1) + ->setParam('collectionId', $collectionId) + ; $audits ->setParam('event', 'database.documents.update') @@ -1467,11 +1810,13 @@ App::delete('/v1/database/collections/:collectionId/documents/:documentId') ->inject('dbForExternal') ->inject('events') ->inject('audits') - ->action(function ($collectionId, $documentId, $response, $dbForInternal, $dbForExternal, $events, $audits) { + ->inject('usage') + ->action(function ($collectionId, $documentId, $response, $dbForInternal, $dbForExternal, $events, $audits, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForExternal */ /** @var Appwrite\Event\Event $events */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ $collection = $dbForInternal->getDocument('collections', $collectionId); @@ -1479,7 +1824,22 @@ App::delete('/v1/database/collections/:collectionId/documents/:documentId') throw new Exception('Collection not found', 404); } - $document = $dbForExternal->getDocument($collectionId, $documentId); + // Check collection permissions when enforced + if ($collection->getAttribute('permission') === 'collection') { + $validator = new Authorization('write'); + if (!$validator->isValid($collection->getWrite())) { + throw new Exception('Unauthorized permissions', 401); + } + } + + if ($collection->getAttribute('permission') === 'collection') { + /** @var Document $document */ + $document = Authorization::skip(function() use ($dbForExternal, $collectionId, $documentId) { + return $dbForExternal->getDocument($collectionId, $documentId); + }); + } else { + $document = $dbForExternal->getDocument($collectionId, $documentId); + } if ($document->isEmpty()) { throw new Exception('No document found', 404); @@ -1487,6 +1847,11 @@ App::delete('/v1/database/collections/:collectionId/documents/:documentId') $dbForExternal->deleteDocument($collectionId, $documentId); + $usage + ->setParam('database.documents.delete', 1) + ->setParam('collectionId', $collectionId) + ; + $events ->setParam('eventData', $response->output($document, Response::MODEL_DOCUMENT)) ; diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index fbd27c47f..00ed18123 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -53,8 +53,9 @@ App::post('/v1/functions') /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ + $functionId = ($functionId == 'unique()') ? $dbForInternal->getId() : $functionId; $function = $dbForInternal->createDocument('functions', new Document([ - '$id' => $functionId == 'unique()' ? $dbForInternal->getId() : $functionId, + '$id' => $functionId, 'execute' => $execute, 'dateCreated' => time(), 'dateUpdated' => time(), @@ -68,6 +69,7 @@ App::post('/v1/functions') 'schedulePrevious' => 0, 'scheduleNext' => 0, 'timeout' => $timeout, + 'search' => implode(' ', [$functionId, $name, $runtime]), ])); $response->setStatusCode(Response::STATUS_CODE_CREATED); @@ -96,8 +98,6 @@ App::get('/v1/functions') /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ - $queries = ($search) ? [new Query('name', Query::TYPE_SEARCH, [$search])] : []; - if (!empty($after)) { $afterFunction = $dbForInternal->getDocument('functions', $after); @@ -105,6 +105,12 @@ App::get('/v1/functions') throw new Exception("Function '{$after}' for the 'after' value not found.", 400); } } + + $queries = []; + + if (!empty($search)) { + $queries[] = new Query('search', Query::TYPE_SEARCH, [$search]); + } $response->dynamic(new Document([ 'functions' => $dbForInternal->find('functions', $queries, $limit, $offset, [], [$orderType], $afterFunction ?? null), @@ -146,6 +152,9 @@ App::get('/v1/functions/:functionId/usage') ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) ->label('sdk.namespace', 'functions') ->label('sdk.method', 'getUsage') + ->label('sdk.response.code', Response::STATUS_CODE_OK) + ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) + ->label('sdk.response.model', Response::MODEL_USAGE_FUNCTIONS) ->param('functionId', '', new UID(), 'Function unique ID.') ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d']), 'Date range.', true) ->inject('response') @@ -164,99 +173,62 @@ App::get('/v1/functions/:functionId/usage') throw new Exception('Function not found', 404); } + $usage = []; if(App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { $period = [ '24h' => [ - 'start' => DateTime::createFromFormat('U', \strtotime('-24 hours')), - 'end' => DateTime::createFromFormat('U', \strtotime('+1 hour')), - 'group' => '30m', + 'period' => '30m', + 'limit' => 48, ], '7d' => [ - 'start' => DateTime::createFromFormat('U', \strtotime('-7 days')), - 'end' => DateTime::createFromFormat('U', \strtotime('now')), - 'group' => '1d', + 'period' => '1d', + 'limit' => 7, ], '30d' => [ - 'start' => DateTime::createFromFormat('U', \strtotime('-30 days')), - 'end' => DateTime::createFromFormat('U', \strtotime('now')), - 'group' => '1d', + 'period' => '1d', + 'limit' => 30, ], '90d' => [ - 'start' => DateTime::createFromFormat('U', \strtotime('-90 days')), - 'end' => DateTime::createFromFormat('U', \strtotime('now')), - 'group' => '1d', + 'period' => '1d', + 'limit' => 90, ], ]; + + $metrics = [ + "functions.$functionId.executions", + "functions.$functionId.failures", + "functions.$functionId.compute" + ]; + + $stats = []; + + Authorization::skip(function() use ($dbForInternal, $period, $range, $metrics, &$stats) { + foreach ($metrics as $metric) { + $requestDocs = $dbForInternal->find('stats', [ + new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]), + new Query('metric', Query::TYPE_EQUAL, [$metric]), + ], $period[$range]['limit'], 0, ['time'], [Database::ORDER_DESC]); - $client = $register->get('influxdb'); - - $executions = []; - $failures = []; - $compute = []; - - if ($client) { - $start = $period[$range]['start']->format(DateTime::RFC3339); - $end = $period[$range]['end']->format(DateTime::RFC3339); - $database = $client->selectDB('telegraf'); - - // Executions - $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_executions_all" WHERE time > \''.$start.'\' AND time < \''.$end.'\' AND "metric_type"=\'counter\' AND "project"=\''.$project->getId().'\' AND "functionId"=\''.$function->getId().'\' GROUP BY time('.$period[$range]['group'].') FILL(null)'); - $points = $result->getPoints(); - - foreach ($points as $point) { - $executions[] = [ - 'value' => (!empty($point['value'])) ? $point['value'] : 0, - 'date' => \strtotime($point['time']), - ]; - } - - // Failures - $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_executions_all" WHERE time > \''.$start.'\' AND time < \''.$end.'\' AND "metric_type"=\'counter\' AND "project"=\''.$project->getId().'\' AND "functionId"=\''.$function->getId().'\' AND "functionStatus"=\'failed\' GROUP BY time('.$period[$range]['group'].') FILL(null)'); - $points = $result->getPoints(); - - foreach ($points as $point) { - $failures[] = [ - 'value' => (!empty($point['value'])) ? $point['value'] : 0, - 'date' => \strtotime($point['time']), - ]; - } - - // Compute - $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_executions_time" WHERE time > \''.$start.'\' AND time < \''.$end.'\' AND "metric_type"=\'counter\' AND "project"=\''.$project->getId().'\' AND "functionId"=\''.$function->getId().'\' GROUP BY time('.$period[$range]['group'].') FILL(null)'); - $points = $result->getPoints(); - - foreach ($points as $point) { - $compute[] = [ - 'value' => round((!empty($point['value'])) ? $point['value'] / 1000 : 0, 2), // minutes - 'date' => \strtotime($point['time']), - ]; - } - } - - $response->json([ + $stats[$metric] = []; + foreach ($requestDocs as $requestDoc) { + $stats[$metric][] = [ + 'value' => $requestDoc->getAttribute('value'), + 'date' => $requestDoc->getAttribute('time'), + ]; + } + $stats[$metric] = array_reverse($stats[$metric]); + } + }); + + $usage = new Document([ 'range' => $range, - 'executions' => [ - 'data' => $executions, - 'total' => \array_sum(\array_map(function ($item) { - return $item['value']; - }, $executions)), - ], - 'failures' => [ - 'data' => $failures, - 'total' => \array_sum(\array_map(function ($item) { - return $item['value']; - }, $failures)), - ], - 'compute' => [ - 'data' => $compute, - 'total' => \array_sum(\array_map(function ($item) { - return $item['value']; - }, $compute)), - ], + 'functions.executions' => $stats["functions.$functionId.executions"], + 'functions.failures' => $stats["functions.$functionId.failures"], + 'functions.compute' => $stats["functions.$functionId.compute"] ]); - } else { - $response->json([]); } + + $response->dynamic($usage, Response::MODEL_USAGE_FUNCTIONS); }); App::put('/v1/functions/:functionId') @@ -305,6 +277,7 @@ App::put('/v1/functions/:functionId') 'schedule' => $schedule, 'scheduleNext' => (int)$next, 'timeout' => $timeout, + 'search' => implode(' ', [$functionId, $name, $function->getAttribute('runtime')]), ]))); if ($next && $schedule !== $original) { @@ -481,8 +454,9 @@ App::post('/v1/functions/:functionId/tags') throw new Exception('Failed moving file', 500); } + $tagId = $dbForInternal->getId(); $tag = $dbForInternal->createDocument('tags', new Document([ - '$id' => $dbForInternal->getId(), + '$id' => $tagId, '$read' => [], '$write' => [], 'functionId' => $function->getId(), @@ -490,6 +464,7 @@ App::post('/v1/functions/:functionId/tags') 'command' => $command, 'path' => $path, 'size' => $size, + 'search' => implode(' ', [$tagId, $command]), ])); $usage @@ -529,8 +504,6 @@ App::get('/v1/functions/:functionId/tags') throw new Exception('Function not found', 404); } - $queries[] = new Query('functionId', Query::TYPE_EQUAL, [$function->getId()]); - if (!empty($after)) { $afterTag = $dbForInternal->getDocument('tags', $after); @@ -539,6 +512,14 @@ App::get('/v1/functions/:functionId/tags') } } + $queries = []; + + if (!empty($search)) { + $queries[] = new Query('search', Query::TYPE_SEARCH, [$search]); + } + + $queries[] = new Query('functionId', Query::TYPE_EQUAL, [$function->getId()]); + $results = $dbForInternal->find('tags', $queries, $limit, $offset, [], [$orderType], $afterTag ?? null); $sum = $dbForInternal->count('tags', $queries, APP_LIMIT_COUNT); @@ -699,8 +680,10 @@ App::post('/v1/functions/:functionId/executions') Authorization::disable(); + $executionId = $dbForInternal->getId(); + $execution = $dbForInternal->createDocument('executions', new Document([ - '$id' => $dbForInternal->getId(), + '$id' => $executionId, '$read' => (!$user->isEmpty()) ? ['user:' . $user->getId()] : [], '$write' => [], 'dateCreated' => time(), @@ -712,6 +695,7 @@ App::post('/v1/functions/:functionId/executions') 'stdout' => '', 'stderr' => '', 'time' => 0.0, + 'search' => implode(' ', [$functionId, $executionId]), ])); Authorization::reset(); @@ -766,10 +750,11 @@ App::get('/v1/functions/:functionId/executions') ->param('functionId', '', new UID(), 'Function unique ID.') ->param('limit', 25, new Range(0, 100), 'Results limit value. By default will return maximum 25 results. Maximum of 100 results allowed per request.', true) ->param('offset', 0, new Range(0, 2000), 'Results offset. The default value is 0. Use this param to manage pagination.', true) + ->param('search', '', new Text(256), 'Search term to filter your list results. Max length: 256 chars.', true) ->param('after', '', new UID(), 'ID of the execution used as the starting point for the query, excluding the execution itself. Should be used for efficient pagination when working with large sets of data.', true) ->inject('response') ->inject('dbForInternal') - ->action(function ($functionId, $limit, $offset, $after, $response, $dbForInternal) { + ->action(function ($functionId, $limit, $offset, $search, $after, $response, $dbForInternal) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ @@ -789,13 +774,17 @@ App::get('/v1/functions/:functionId/executions') } } - $results = $dbForInternal->find('executions', [ - new Query('functionId', Query::TYPE_EQUAL, [$function->getId()]), - ], $limit, $offset, [], [Database::ORDER_DESC], $afterExecution ?? null); + $queries = [ + new Query('functionId', Query::TYPE_EQUAL, [$function->getId()]) + ]; - $sum = $dbForInternal->count('executions', [ - new Query('functionId', Query::TYPE_EQUAL, [$function->getId()]), - ], APP_LIMIT_COUNT); + if (!empty($search)) { + $queries[] = new Query('search', Query::TYPE_SEARCH, [$search]); + } + + $results = $dbForInternal->find('executions', $queries, $limit, $offset, [], [Database::ORDER_DESC], $afterExecution ?? null); + + $sum = $dbForInternal->count('executions', $queries, APP_LIMIT_COUNT); $response->dynamic(new Document([ 'executions' => $results, diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index b83dff6d9..bc20bfdea 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -6,10 +6,15 @@ use Appwrite\Network\Validator\CNAME; use Appwrite\Network\Validator\Domain as DomainValidator; use Appwrite\Network\Validator\URL; use Appwrite\Utopia\Response; +use Utopia\Abuse\Adapters\TimeLimit; use Utopia\App; +use Utopia\CLI\CLI; +use Utopia\Audit\Audit; use Utopia\Config\Config; +use Utopia\Database\Database; use Utopia\Database\Document; use Utopia\Database\Query; +use Utopia\Database\Validator\Authorization; use Utopia\Database\Validator\UID; use Utopia\Domains\Domain; use Utopia\Exception; @@ -19,13 +24,11 @@ use Utopia\Validator\Integer; use Utopia\Validator\Range; use Utopia\Validator\Text; use Utopia\Validator\WhiteList; -use Utopia\Audit\Audit; -use Utopia\Abuse\Adapters\TimeLimit; App::init(function ($project) { /** @var Utopia\Database\Document $project */ - if($project->getId() !== 'console') { + if ($project->getId() !== 'console') { throw new Exception('Access to this API is forbidden.', 401); } }, ['project'], 'projects'); @@ -69,13 +72,14 @@ App::post('/v1/projects') if ($team->isEmpty()) { throw new Exception('Team not found', 404); } - + $auth = Config::getParam('auth', []); $auths = ['limit' => 0]; foreach ($auth as $index => $method) { $auths[$method['key'] ?? ''] = true; } + $projectId = ($projectId == 'unique()') ? $dbForConsole->getId() : $projectId; $project = $dbForConsole->createDocument('projects', new Document([ '$id' => $projectId == 'unique()' ? $dbForConsole->getId() : $projectId, '$read' => ['team:' . $teamId], @@ -93,11 +97,13 @@ App::post('/v1/projects') 'legalAddress' => $legalAddress, 'legalTaxId' => $legalTaxId, 'services' => new stdClass(), - 'platforms' => [], - 'webhooks' => [], - 'keys' => [], - 'domains' => [], + 'platforms' => null, + 'providers' => [], + 'webhooks' => null, + 'keys' => null, + 'domains' => null, 'auths' => $auths, + 'search' => implode(' ', [$projectId, $name]), ])); $collections = Config::getParam('collections2', []); /** @var array $collections */ @@ -110,7 +116,7 @@ App::post('/v1/projects') $audit = new Audit($dbForInternal); $audit->setup(); - $adapter = new TimeLimit("", 0, 1, $dbForInternal); + $adapter = new TimeLimit('', 0, 1, $dbForInternal); $adapter->setup(); foreach ($collections as $key => $collection) { @@ -169,8 +175,6 @@ App::get('/v1/projects') /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForConsole */ - $queries = ($search) ? [new Query('name', Query::TYPE_SEARCH, [$search])] : []; - if (!empty($after)) { $afterProject = $dbForConsole->getDocument('projects', $after); @@ -179,6 +183,12 @@ App::get('/v1/projects') } } + $queries = []; + + if (!empty($search)) { + $queries[] = new Query('search', Query::TYPE_SEARCH, [$search]); + } + $results = $dbForConsole->find('projects', $queries, $limit, $offset, [], [$orderType], $afterProject ?? null); $sum = $dbForConsole->count('projects', $queries, APP_LIMIT_COUNT); @@ -215,22 +225,25 @@ App::get('/v1/projects/:projectId') }); App::get('/v1/projects/:projectId/usage') - ->desc('Get Project') + ->desc('Get usage stats for a project') ->groups(['api', 'projects']) ->label('scope', 'projects.read') ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) ->label('sdk.namespace', 'projects') ->label('sdk.method', 'getUsage') + ->label('sdk.response.code', Response::STATUS_CODE_OK) + ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) + ->label('sdk.response.model', Response::MODEL_USAGE_PROJECT) ->param('projectId', '', new UID(), 'Project unique ID.') ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true) ->inject('response') ->inject('dbForConsole') - ->inject('projectDB') + ->inject('dbForInternal') ->inject('register') - ->action(function ($projectId, $range, $response, $dbForConsole, $projectDB, $register) { + ->action(function ($projectId, $range, $response, $dbForConsole, $dbForInternal, $register) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForConsole */ - /** @var Appwrite\Database\Database $projectDB */ + /** @var Utopia\Database\Database $dbForInternal */ /** @var Utopia\Registry\Registry $register */ $project = $dbForConsole->getDocument('projects', $projectId); @@ -239,172 +252,72 @@ App::get('/v1/projects/:projectId/usage') throw new Exception('Project not found', 404); } + $usage = []; if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { - $period = [ '24h' => [ - 'start' => DateTime::createFromFormat('U', \strtotime('-24 hours')), - 'end' => DateTime::createFromFormat('U', \strtotime('+1 hour')), - 'group' => '30m', + 'period' => '30m', + 'limit' => 48, ], '7d' => [ - 'start' => DateTime::createFromFormat('U', \strtotime('-7 days')), - 'end' => DateTime::createFromFormat('U', \strtotime('now')), - 'group' => '1d', + 'period' => '1d', + 'limit' => 7, ], '30d' => [ - 'start' => DateTime::createFromFormat('U', \strtotime('-30 days')), - 'end' => DateTime::createFromFormat('U', \strtotime('now')), - 'group' => '1d', + 'period' => '1d', + 'limit' => 30, ], '90d' => [ - 'start' => DateTime::createFromFormat('U', \strtotime('-90 days')), - 'end' => DateTime::createFromFormat('U', \strtotime('now')), - 'group' => '1d', + 'period' => '1d', + 'limit' => 90, ], ]; - $client = $register->get('influxdb'); + $dbForInternal->setNamespace('project_' . $projectId . '_internal'); - $requests = []; - $network = []; - $functions = []; + $metrics = [ + 'requests', + 'network', + 'executions', + 'users.count', + 'database.documents.count', + 'database.collections.count', + 'storage.total' + ]; - if ($client) { - $start = $period[$range]['start']->format(DateTime::RFC3339); - $end = $period[$range]['end']->format(DateTime::RFC3339); - $database = $client->selectDB('telegraf'); + $stats = []; - // Requests - $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_requests_all" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' AND "project"=\'' . $project->getId() . '\' GROUP BY time(' . $period[$range]['group'] . ') FILL(null)'); - $points = $result->getPoints(); + Authorization::skip(function() use ($dbForInternal, $period, $range, $metrics, &$stats) { + foreach ($metrics as $metric) { + $requestDocs = $dbForInternal->find('stats', [ + new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]), + new Query('metric', Query::TYPE_EQUAL, [$metric]), + ], $period[$range]['limit'], 0, ['time'], [Database::ORDER_DESC]); + + $stats[$metric] = []; + foreach ($requestDocs as $requestDoc) { + $stats[$metric][] = [ + 'value' => $requestDoc->getAttribute('value'), + 'date' => $requestDoc->getAttribute('time'), + ]; + } + $stats[$metric] = array_reverse($stats[$metric]); + } + }); - foreach ($points as $point) { - $requests[] = [ - 'value' => (!empty($point['value'])) ? $point['value'] : 0, - 'date' => \strtotime($point['time']), - ]; - } - - // Network - $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_network_all" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' AND "project"=\'' . $project->getId() . '\' GROUP BY time(' . $period[$range]['group'] . ') FILL(null)'); - $points = $result->getPoints(); - - foreach ($points as $point) { - $network[] = [ - 'value' => (!empty($point['value'])) ? $point['value'] : 0, - 'date' => \strtotime($point['time']), - ]; - } - - // Functions - $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_executions_all" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' AND "project"=\'' . $project->getId() . '\' GROUP BY time(' . $period[$range]['group'] . ') FILL(null)'); - $points = $result->getPoints(); - - foreach ($points as $point) { - $functions[] = [ - 'value' => (!empty($point['value'])) ? $point['value'] : 0, - 'date' => \strtotime($point['time']), - ]; - } - } - } else { - $requests = []; - $network = []; - $functions = []; - } - - // Users - - $projectDB->getCollection([ - 'limit' => 0, - 'offset' => 0, - 'filters' => [ - '$collection=users', - ], - ]); - - $usersTotal = $projectDB->getSum(); - - // Documents - - $collections = $projectDB->getCollection([ - 'limit' => 100, - 'offset' => 0, - 'filters' => [ - '$collection=collections', - ], - ]); - - $collectionsTotal = $projectDB->getSum(); - - $documents = []; - - foreach ($collections as $collection) { - $result = $projectDB->getCollection([ - 'limit' => 0, - 'offset' => 0, - 'filters' => [ - '$collection=' . $collection['$id'], - ], + $usage = new Document([ + 'range' => $range, + 'requests' => $stats['requests'], + 'network' => $stats['network'], + 'functions' => $stats['executions'], + 'documents' => $stats['database.documents.count'], + 'collections' => $stats['database.collections.count'], + 'users' => $stats['users.count'], + 'storage' => $stats['storage.total'] ]); - - $documents[] = ['name' => $collection['name'], 'total' => $projectDB->getSum()]; } - $response->json([ - 'range' => $range, - 'requests' => [ - 'data' => $requests, - 'total' => \array_sum(\array_map(function ($item) { - return $item['value']; - }, $requests)), - ], - 'network' => [ - 'data' => \array_map(function ($value) {return ['value' => \round($value['value'] / 1000000, 2), 'date' => $value['date']];}, $network), // convert bytes to mb - 'total' => \array_sum(\array_map(function ($item) { - return $item['value']; - }, $network)), - ], - 'functions' => [ - 'data' => $functions, - 'total' => \array_sum(\array_map(function ($item) { - return $item['value']; - }, $functions)), - ], - 'collections' => [ - 'data' => $collections, - 'total' => $collectionsTotal, - ], - 'documents' => [ - 'data' => $documents, - 'total' => \array_sum(\array_map(function ($item) { - return $item['total']; - }, $documents)), - ], - 'users' => [ - 'data' => [], - 'total' => $usersTotal, - ], - 'storage' => [ - 'total' => $projectDB->getCount( - [ - 'attribute' => 'sizeOriginal', - 'filters' => [ - '$collection=files', - ], - ] - ) + - $projectDB->getCount( - [ - 'attribute' => 'size', - 'filters' => [ - '$collection=tags', - ], - ] - ), - ], - ]); + $response->dynamic($usage, Response::MODEL_USAGE_PROJECT); }); App::patch('/v1/projects/:projectId') @@ -451,6 +364,7 @@ App::patch('/v1/projects/:projectId') ->setAttribute('legalCity', $legalCity) ->setAttribute('legalAddress', $legalAddress) ->setAttribute('legalTaxId', $legalTaxId) + ->setAttribute('search', implode(' ', [$projectId, $name])) ); $response->dynamic($project, Response::MODEL_PROJECT); @@ -467,13 +381,14 @@ App::patch('/v1/projects/:projectId/service') ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_PROJECT) ->param('projectId', '', new UID(), 'Project unique ID.') - ->param('service', '', new WhiteList(array_keys(array_filter(Config::getParam('services'), function($element) {return $element['optional'];})), true), 'Service name.') + ->param('service', '', new WhiteList(array_keys(array_filter(Config::getParam('services'), function ($element) {return $element['optional'];})), true), 'Service name.') ->param('status', null, new Boolean(), 'Service status.') ->inject('response') ->inject('dbForConsole') ->action(function ($projectId, $service, $status, $response, $dbForConsole) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForConsole */ + /** @var Boolean $status */ $project = $dbForConsole->getDocument('projects', $projectId); @@ -552,7 +467,7 @@ App::patch('/v1/projects/:projectId/auth/limit') $auths['limit'] = $limit; $dbForConsole->updateDocument('projects', $project->getId(), $project - ->setAttribute('auths', $auths) + ->setAttribute('auths', $auths) ); $response->dynamic($project, Response::MODEL_PROJECT); @@ -676,6 +591,9 @@ App::post('/v1/projects/:projectId/webhooks') $webhook = new Document([ '$id' => $dbForConsole->getId(), + '$read' => ['role:all'], + '$write' => ['role:all'], + 'projectId' => $project->getId(), 'name' => $name, 'events' => $events, 'url' => $url, @@ -684,9 +602,9 @@ App::post('/v1/projects/:projectId/webhooks') 'httpPass' => $httpPass, ]); - $project = $dbForConsole->updateDocument('projects', $project->getId(), $project - ->setAttribute('webhooks', $webhook, Document::SET_TYPE_APPEND) - ); + $webhook = $dbForConsole->createDocument('webhooks', $webhook); + + $dbForConsole->deleteCachedDocument('projects', $project->getId()); $response->setStatusCode(Response::STATUS_CODE_CREATED); $response->dynamic($webhook, Response::MODEL_WEBHOOK); @@ -715,7 +633,9 @@ App::get('/v1/projects/:projectId/webhooks') throw new Exception('Project not found', 404); } - $webhooks = $project->getAttribute('webhooks', []); + $webhooks = $dbForConsole->find('webhooks', [ + new Query('projectId', Query::TYPE_EQUAL, [$project->getId()]) + ]); $response->dynamic(new Document([ 'webhooks' => $webhooks, @@ -747,9 +667,12 @@ App::get('/v1/projects/:projectId/webhooks/:webhookId') throw new Exception('Project not found', 404); } - $webhook = $project->find('$id', $webhookId, 'webhooks'); + $webhook = $dbForConsole->findOne('webhooks', [ + new Query('_uid', Query::TYPE_EQUAL, [$webhookId]), + new Query('projectId', Query::TYPE_EQUAL, [$project->getId()]) + ]); - if (empty($webhook) || !$webhook instanceof Document) { + if ($webhook === false || $webhook->isEmpty()) { throw new Exception('Webhook not found', 404); } @@ -788,22 +711,27 @@ App::put('/v1/projects/:projectId/webhooks/:webhookId') $security = ($security === '1' || $security === 'true' || $security === 1 || $security === true); - $webhook = $project->find('$id', $webhookId, 'webhooks'); + $webhook = $dbForConsole->findOne('webhooks', [ + new Query('_uid', Query::TYPE_EQUAL, [$webhookId]), + new Query('projectId', Query::TYPE_EQUAL, [$project->getId()]) + ]); - if (empty($webhook) || !$webhook instanceof Document) { + if ($webhook === false || $webhook->isEmpty()) { throw new Exception('Webhook not found', 404); } - $project->findAndReplace('$id', $webhook->getId(), $webhook - ->setAttribute('name', $name) - ->setAttribute('events', $events) - ->setAttribute('url', $url) - ->setAttribute('security', $security) - ->setAttribute('httpUser', $httpUser) - ->setAttribute('httpPass', $httpPass) - , 'webhooks'); + $webhook + ->setAttribute('name', $name) + ->setAttribute('events', $events) + ->setAttribute('url', $url) + ->setAttribute('security', $security) + ->setAttribute('httpUser', $httpUser) + ->setAttribute('httpPass', $httpPass) + ; - $dbForConsole->updateDocument('projects', $project->getId(), $project); + $dbForConsole->updateDocument('webhooks', $webhook->getId(), $webhook); + + $dbForConsole->deleteCachedDocument('projects', $project->getId()); $response->dynamic($webhook, Response::MODEL_WEBHOOK); }); @@ -831,11 +759,18 @@ App::delete('/v1/projects/:projectId/webhooks/:webhookId') throw new Exception('Project not found', 404); } - if (!$project->findAndRemove('$id', $webhookId, 'webhooks')) { + $webhook = $dbForConsole->findOne('webhooks', [ + new Query('_uid', Query::TYPE_EQUAL, [$webhookId]), + new Query('projectId', Query::TYPE_EQUAL, [$project->getId()]) + ]); + + if($webhook === false || $webhook->isEmpty()) { throw new Exception('Webhook not found', 404); } - $dbForConsole->updateDocument('projects', $project->getId(), $project); + $dbForConsole->deleteDocument('webhooks', $webhook->getId()); + + $dbForConsole->deleteCachedDocument('projects', $project->getId()); $response->noContent(); }); @@ -869,14 +804,17 @@ App::post('/v1/projects/:projectId/keys') $key = new Document([ '$id' => $dbForConsole->getId(), + '$read' => ['role:all'], + '$write' => ['role:all'], + 'projectId' => $project->getId(), 'name' => $name, 'scopes' => $scopes, 'secret' => \bin2hex(\random_bytes(128)), ]); - $project = $dbForConsole->updateDocument('projects', $project->getId(), $project - ->setAttribute('keys', $key, Document::SET_TYPE_APPEND) - ); + $key = $dbForConsole->createDocument('keys', $key); + + $dbForConsole->deleteCachedDocument('projects', $project->getId()); $response->setStatusCode(Response::STATUS_CODE_CREATED); $response->dynamic($key, Response::MODEL_KEY); @@ -905,7 +843,9 @@ App::get('/v1/projects/:projectId/keys') throw new Exception('Project not found', 404); } - $keys = $project->getAttribute('keys', []); + $keys = $dbForConsole->find('keys', [ + new Query('projectId', Query::TYPE_EQUAL, [$project->getId()]), + ], 5000); $response->dynamic(new Document([ 'keys' => $keys, @@ -928,15 +868,21 @@ App::get('/v1/projects/:projectId/keys/:keyId') ->inject('response') ->inject('dbForConsole') ->action(function ($projectId, $keyId, $response, $dbForConsole) { + /** @var Appwrite\Utopia\Response $response */ + /** @var Utopia\Database\Database $dbForConsole */ + $project = $dbForConsole->getDocument('projects', $projectId); if ($project->isEmpty()) { throw new Exception('Project not found', 404); } - $key = $project->find('$id', $keyId, 'keys'); + $key = $dbForConsole->findOne('keys', [ + new Query('_uid', Query::TYPE_EQUAL, [$keyId]), + new Query('projectId', Query::TYPE_EQUAL, [$project->getId()]) + ]); - if (empty($key) || !$key instanceof Document) { + if ($key === false || $key->isEmpty()) { throw new Exception('Key not found', 404); } @@ -969,18 +915,23 @@ App::put('/v1/projects/:projectId/keys/:keyId') throw new Exception('Project not found', 404); } - $key = $project->find('$id', $keyId, 'keys'); + $key = $dbForConsole->findOne('keys', [ + new Query('_uid', Query::TYPE_EQUAL, [$keyId]), + new Query('projectId', Query::TYPE_EQUAL, [$project->getId()]) + ]); - if (empty($key) || !$key instanceof Document) { + if ($key === false || $key->isEmpty()) { throw new Exception('Key not found', 404); } - $project->findAndReplace('$id', $key->getId(), $key - ->setAttribute('name', $name) - ->setAttribute('scopes', $scopes) - , 'keys'); + $key + ->setAttribute('name', $name) + ->setAttribute('scopes', $scopes) + ; - $dbForConsole->updateDocument('projects', $project->getId(), $project); + $dbForConsole->updateDocument('keys', $key->getId(), $key); + + $dbForConsole->deleteCachedDocument('projects', $project->getId()); $response->dynamic($key, Response::MODEL_KEY); }); @@ -1008,11 +959,18 @@ App::delete('/v1/projects/:projectId/keys/:keyId') throw new Exception('Project not found', 404); } - if (!$project->findAndRemove('$id', $keyId, 'keys')) { + $key = $dbForConsole->findOne('keys', [ + new Query('_uid', Query::TYPE_EQUAL, [$keyId]), + new Query('projectId', Query::TYPE_EQUAL, [$project->getId()]) + ]); + + if($key === false || $key->isEmpty()) { throw new Exception('Key not found', 404); } - $dbForConsole->updateDocument('projects', $project->getId(), $project); + $dbForConsole->deleteDocument('keys', $key->getId()); + + $dbForConsole->deleteCachedDocument('projects', $project->getId()); $response->noContent(); }); @@ -1049,6 +1007,9 @@ App::post('/v1/projects/:projectId/platforms') $platform = new Document([ '$id' => $dbForConsole->getId(), + '$read' => ['role:all'], + '$write' => ['role:all'], + 'projectId' => $project->getId(), 'type' => $type, 'name' => $name, 'key' => $key, @@ -1058,9 +1019,9 @@ App::post('/v1/projects/:projectId/platforms') 'dateUpdated' => \time(), ]); - $project = $dbForConsole->updateDocument('projects', $project->getId(), $project - ->setAttribute('platforms', $platform, Document::SET_TYPE_APPEND) - ); + $platform = $dbForConsole->createDocument('platforms', $platform); + + $dbForConsole->deleteCachedDocument('projects', $project->getId()); $response->setStatusCode(Response::STATUS_CODE_CREATED); $response->dynamic($platform, Response::MODEL_PLATFORM); @@ -1089,7 +1050,9 @@ App::get('/v1/projects/:projectId/platforms') throw new Exception('Project not found', 404); } - $platforms = $project->getAttribute('platforms', []); + $platforms = $dbForConsole->find('platforms', [ + new Query('projectId', Query::TYPE_EQUAL, [$project->getId()]) + ], 5000); $response->dynamic(new Document([ 'platforms' => $platforms, @@ -1121,9 +1084,12 @@ App::get('/v1/projects/:projectId/platforms/:platformId') throw new Exception('Project not found', 404); } - $platform = $project->find('$id', $platformId, 'platforms'); + $platform = $dbForConsole->findOne('platforms', [ + new Query('_uid', Query::TYPE_EQUAL, [$platformId]), + new Query('projectId', Query::TYPE_EQUAL, [$project->getId()]) + ]); - if (empty($platform) || !$platform instanceof Document) { + if ($platform === false || $platform->isEmpty()) { throw new Exception('Platform not found', 404); } @@ -1158,9 +1124,12 @@ App::put('/v1/projects/:projectId/platforms/:platformId') throw new Exception('Project not found', 404); } - $platform = $project->find('$id', $platformId, 'platforms'); + $platform = $dbForConsole->findOne('platforms', [ + new Query('_uid', Query::TYPE_EQUAL, [$platformId]), + new Query('projectId', Query::TYPE_EQUAL, [$project->getId()]) + ]); - if (empty($platform) || !$platform instanceof Document) { + if ($platform === false || $platform->isEmpty()) { throw new Exception('Platform not found', 404); } @@ -1172,15 +1141,9 @@ App::put('/v1/projects/:projectId/platforms/:platformId') ->setAttribute('hostname', $hostname) ; - $project->findAndReplace('$id', $platform->getId(), $platform - ->setAttribute('name', $name) - ->setAttribute('dateUpdated', \time()) - ->setAttribute('key', $key) - ->setAttribute('store', $store) - ->setAttribute('hostname', $hostname) - , 'platforms'); + $dbForConsole->updateDocument('platforms', $platform->getId(), $platform); - $dbForConsole->updateDocument('projects', $project->getId(), $project); + $dbForConsole->deleteCachedDocument('projects', $project->getId()); $response->dynamic($platform, Response::MODEL_PLATFORM); }); @@ -1208,11 +1171,18 @@ App::delete('/v1/projects/:projectId/platforms/:platformId') throw new Exception('Project not found', 404); } - if (!$project->findAndRemove('$id', $platformId, 'platforms')) { + $platform = $dbForConsole->findOne('platforms', [ + new Query('_uid', Query::TYPE_EQUAL, [$platformId]), + new Query('projectId', Query::TYPE_EQUAL, [$project->getId()]) + ]); + + if ($platform === false || $platform->isEmpty()) { throw new Exception('Platform not found', 404); } - $dbForConsole->updateDocument('projects', $project->getId(), $project); + $dbForConsole->deleteDocument('platforms', $platformId); + + $dbForConsole->deleteCachedDocument('projects', $project->getId()); $response->noContent(); }); @@ -1243,9 +1213,12 @@ App::post('/v1/projects/:projectId/domains') throw new Exception('Project not found', 404); } - $document = $project->find('domain', $domain, 'domains'); + $document = $dbForConsole->findOne('domains', [ + new Query('domain', Query::TYPE_EQUAL, [$domain]), + new Query('projectId', Query::TYPE_EQUAL, [$project->getId()]), + ]); - if ($document) { + if ($document && !$document->isEmpty()) { throw new Exception('Domain already exists', 409); } @@ -1259,6 +1232,9 @@ App::post('/v1/projects/:projectId/domains') $domain = new Document([ '$id' => $dbForConsole->getId(), + '$read' => ['role:all'], + '$write' => ['role:all'], + 'projectId' => $project->getId(), 'updated' => \time(), 'domain' => $domain->get(), 'tld' => $domain->getSuffix(), @@ -1267,9 +1243,9 @@ App::post('/v1/projects/:projectId/domains') 'certificateId' => null, ]); - $project = $dbForConsole->updateDocument('projects', $project->getId(), $project - ->setAttribute('domains', $domain, Document::SET_TYPE_APPEND) - ); + $domain = $dbForConsole->createDocument('domains', $domain); + + $dbForConsole->deleteCachedDocument('projects', $project->getId()); $response->setStatusCode(Response::STATUS_CODE_CREATED); $response->dynamic($domain, Response::MODEL_DOMAIN); @@ -1298,7 +1274,9 @@ App::get('/v1/projects/:projectId/domains') throw new Exception('Project not found', 404); } - $domains = $project->getAttribute('domains', []); + $domains = $dbForConsole->find('domains', [ + new Query('projectId', Query::TYPE_EQUAL, [$project->getId()]) + ], 5000); $response->dynamic(new Document([ 'domains' => $domains, @@ -1330,9 +1308,12 @@ App::get('/v1/projects/:projectId/domains/:domainId') throw new Exception('Project not found', 404); } - $domain = $project->find('$id', $domainId, 'domains'); + $domain = $dbForConsole->findOne('domains', [ + new Query('_uid', Query::TYPE_EQUAL, [$domainId]), + new Query('projectId', Query::TYPE_EQUAL, [$project->getId()]) + ]); - if (empty($domain) || !$domain instanceof Document) { + if ($domain === false || $domain->isEmpty()) { throw new Exception('Domain not found', 404); } @@ -1363,9 +1344,12 @@ App::patch('/v1/projects/:projectId/domains/:domainId/verification') throw new Exception('Project not found', 404); } - $domain = $project->find('$id', $domainId, 'domains'); + $domain = $dbForConsole->findOne('domains', [ + new Query('_uid', Query::TYPE_EQUAL, [$domainId]), + new Query('projectId', Query::TYPE_EQUAL, [$project->getId()]) + ]); - if (empty($domain) || !$domain instanceof Document) { + if ($domain === false || $domain->isEmpty()) { throw new Exception('Domain not found', 404); } @@ -1385,11 +1369,9 @@ App::patch('/v1/projects/:projectId/domains/:domainId/verification') throw new Exception('Failed to verify domain', 401); } - $project->findAndReplace('$id', $domain->getId(), $domain - ->setAttribute('verification', true) - , 'domains'); - $dbForConsole->updateDocument('projects', $project->getId(), $project); + $dbForConsole->updateDocument('domains', $domain->getId(), $domain->setAttribute('verification', true)); + $dbForConsole->deleteCachedDocument('projects', $project->getId()); // Issue a TLS certificate when domain is verified Resque::enqueue('v1-certificates', 'CertificatesV1', [ @@ -1424,13 +1406,18 @@ App::delete('/v1/projects/:projectId/domains/:domainId') throw new Exception('Project not found', 404); } - $domain = $project->find('$id', $domainId, 'domains'); + $domain = $dbForConsole->findOne('domains', [ + new Query('_uid', Query::TYPE_EQUAL, [$domainId]), + new Query('projectId', Query::TYPE_EQUAL, [$project->getId()]) + ]); - if (!$project->findAndRemove('$id', $domainId, 'domains')) { + if ($domain === false || $domain->isEmpty()) { throw new Exception('Domain not found', 404); } - $dbForConsole->updateDocument('projects', $project->getId(), $project); + $dbForConsole->deleteDocument('domains', $domain->getId()); + + $dbForConsole->deleteCachedDocument('projects', $project->getId()); $deletes ->setParam('type', DELETE_TYPE_CERTIFICATES) diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index af58a010d..6c9449452 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -10,6 +10,7 @@ use Utopia\Validator\HexColor; use Utopia\Cache\Cache; use Utopia\Cache\Adapter\Filesystem; use Appwrite\ClamAV\Network; +use Utopia\Database\Validator\Authorization; use Appwrite\Database\Validator\CustomId; use Utopia\Database\Document; use Utopia\Database\Validator\UID; @@ -22,6 +23,7 @@ use Utopia\Image\Image; use Appwrite\OpenSSL\OpenSSL; use Appwrite\Utopia\Response; use Utopia\Config\Config; +use Utopia\Database\Database; use Utopia\Database\Query; App::post('/v1/storage/files') @@ -54,7 +56,7 @@ App::post('/v1/storage/files') /** @var Utopia\Database\Database $dbForInternal */ /** @var Utopia\Database\Document $user */ /** @var Appwrite\Event\Event $audits */ - /** @var Appwrite\Event\Event $usage */ + /** @var Appwrite\Stats\Stats $usage */ $file = $request->getFiles('file'); @@ -123,13 +125,14 @@ App::post('/v1/storage/files') $sizeActual = $device->getFileSize($path); + $fileId = ($fileId == 'unique()') ? $dbForInternal->getId() : $fileId; $file = $dbForInternal->createDocument('files', new Document([ - '$id' => $fileId == 'unique()' ? $dbForInternal->getId() : $fileId, + '$id' => $fileId, '$read' => (is_null($read) && !$user->isEmpty()) ? ['user:'.$user->getId()] : $read ?? [], // By default set read permissions for user '$write' => (is_null($write) && !$user->isEmpty()) ? ['user:'.$user->getId()] : $write ?? [], // By default set write permissions for user 'dateCreated' => \time(), 'bucketId' => '', - 'name' => $file['name'], + 'name' => $file['name'] ?? '', 'path' => $path, 'signature' => $device->getFileHash($path), 'mimeType' => $mimeType, @@ -141,6 +144,7 @@ App::post('/v1/storage/files') 'openSSLCipher' => OpenSSL::CIPHER_AES_128_GCM, 'openSSLTag' => \bin2hex($tag), 'openSSLIV' => \bin2hex($iv), + 'search' => implode(' ', [$fileId, $file['name'] ?? '',]), ])); $audits @@ -150,6 +154,8 @@ App::post('/v1/storage/files') $usage ->setParam('storage', $sizeActual) + ->setParam('storage.files.create', 1) + ->setParam('bucketId', 'default') ; $response->setStatusCode(Response::STATUS_CODE_CREATED); @@ -175,11 +181,11 @@ App::get('/v1/storage/files') ->param('orderType', 'ASC', new WhiteList(['ASC', 'DESC'], true), 'Order result by ASC or DESC order.', true) ->inject('response') ->inject('dbForInternal') - ->action(function ($search, $limit, $offset, $after, $orderType, $response, $dbForInternal) { + ->inject('usage') + ->action(function ($search, $limit, $offset, $after, $orderType, $response, $dbForInternal, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ - - $queries = ($search) ? [new Query('name', Query::TYPE_SEARCH, $search)] : []; + /** @var Appwrite\Stats\Stats $usage */ if (!empty($after)) { $afterFile = $dbForInternal->getDocument('files', $after); @@ -189,6 +195,17 @@ App::get('/v1/storage/files') } } + $queries = []; + + if (!empty($search)) { + $queries[] = new Query('search', Query::TYPE_SEARCH, [$search]); + } + + $usage + ->setParam('storage.files.read', 1) + ->setParam('bucketId', 'default') + ; + $response->dynamic(new Document([ 'files' => $dbForInternal->find('files', $queries, $limit, $offset, [], [$orderType], $afterFile ?? null), 'sum' => $dbForInternal->count('files', $queries, APP_LIMIT_COUNT), @@ -209,16 +226,21 @@ App::get('/v1/storage/files/:fileId') ->param('fileId', '', new UID(), 'File unique ID.') ->inject('response') ->inject('dbForInternal') - ->action(function ($fileId, $response, $dbForInternal) { + ->inject('usage') + ->action(function ($fileId, $response, $dbForInternal, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ + /** @var Appwrite\Stats\Stats $usage */ $file = $dbForInternal->getDocument('files', $fileId); if (empty($file->getId())) { throw new Exception('File not found', 404); } - + $usage + ->setParam('storage.files.read', 1) + ->setParam('bucketId', 'default') + ; $response->dynamic($file, Response::MODEL_FILE); }); @@ -249,11 +271,13 @@ App::get('/v1/storage/files/:fileId/preview') ->inject('response') ->inject('project') ->inject('dbForInternal') - ->action(function ($fileId, $width, $height, $gravity, $quality, $borderWidth, $borderColor, $borderRadius, $opacity, $rotation, $background, $output, $request, $response, $project, $dbForInternal) { + ->inject('usage') + ->action(function ($fileId, $width, $height, $gravity, $quality, $borderWidth, $borderColor, $borderRadius, $opacity, $rotation, $background, $output, $request, $response, $project, $dbForInternal, $usage) { /** @var Utopia\Swoole\Request $request */ /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Document $project */ /** @var Utopia\Database\Database $dbForInternal */ + /** @var Appwrite\Stats\Stats $stats */ $storage = 'files'; @@ -366,6 +390,11 @@ App::get('/v1/storage/files/:fileId/preview') $cache->save($key, $data); + $usage + ->setParam('storage.files.read', 1) + ->setParam('bucketId', 'default') + ; + $response ->setContentType($outputs[$output]) ->addHeader('Expires', $date) @@ -390,9 +419,11 @@ App::get('/v1/storage/files/:fileId/download') ->param('fileId', '', new UID(), 'File unique ID.') ->inject('response') ->inject('dbForInternal') - ->action(function ($fileId, $response, $dbForInternal) { + ->inject('usage') + ->action(function ($fileId, $response, $dbForInternal, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ + /** @var Appwrite\Stats\Stats $usage */ $file = $dbForInternal->getDocument('files', $fileId); @@ -424,6 +455,11 @@ App::get('/v1/storage/files/:fileId/download') $source = $compressor->decompress($source); + $usage + ->setParam('storage.files.read', 1) + ->setParam('bucketId', 'default') + ; + // Response $response ->setContentType($file->getAttribute('mimeType')) @@ -448,9 +484,11 @@ App::get('/v1/storage/files/:fileId/view') ->param('fileId', '', new UID(), 'File unique ID.') ->inject('response') ->inject('dbForInternal') - ->action(function ($fileId, $response, $dbForInternal) { + ->inject('usage') + ->action(function ($fileId, $response, $dbForInternal, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ + /** @var Appwrite\Stats\Stats $usage */ $file = $dbForInternal->getDocument('files', $fileId); $mimes = Config::getParam('storage-mimes'); @@ -490,6 +528,11 @@ App::get('/v1/storage/files/:fileId/view') $output = $compressor->decompress($source); $fileName = $file->getAttribute('name', ''); + $usage + ->setParam('storage.files.read', 1) + ->setParam('bucketId', 'default') + ; + // Response $response ->setContentType($contentType) @@ -520,7 +563,8 @@ App::put('/v1/storage/files/:fileId') ->inject('response') ->inject('dbForInternal') ->inject('audits') - ->action(function ($fileId, $read, $write, $response, $dbForInternal, $audits) { + ->inject('usage') + ->action(function ($fileId, $read, $write, $response, $dbForInternal, $audits, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ /** @var Appwrite\Event\Event $audits */ @@ -542,6 +586,11 @@ App::put('/v1/storage/files/:fileId') ->setParam('resource', 'file/'.$file->getId()) ; + $usage + ->setParam('storage.files.update', 1) + ->setParam('bucketId', 'default') + ; + $response->dynamic($file, Response::MODEL_FILE); }); @@ -567,7 +616,7 @@ App::delete('/v1/storage/files/:fileId') /** @var Utopia\Database\Database $dbForInternal */ /** @var Appwrite\Event\Event $events */ /** @var Appwrite\Event\Event $audits */ - /** @var Appwrite\Event\Event $usage */ + /** @var Appwrite\Stats\Stats $usage */ $file = $dbForInternal->getDocument('files', $fileId); @@ -590,6 +639,8 @@ App::delete('/v1/storage/files/:fileId') $usage ->setParam('storage', $file->getAttribute('size', 0) * -1) + ->setParam('storage.files.delete', 1) + ->setParam('bucketId', 'default') ; $events @@ -597,4 +648,159 @@ App::delete('/v1/storage/files/:fileId') ; $response->noContent(); + }); + +App::get('/v1/storage/usage') + ->desc('Get usage stats for storage') + ->groups(['api', 'storage']) + ->label('scope', 'files.read') + ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) + ->label('sdk.namespace', 'storage') + ->label('sdk.method', 'getUsage') + ->label('sdk.response.code', Response::STATUS_CODE_OK) + ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) + ->label('sdk.response.model', Response::MODEL_USAGE_STORAGE) + ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true) + ->inject('response') + ->inject('dbForInternal') + ->action(function ($range, $response, $dbForInternal) { + /** @var Appwrite\Utopia\Response $response */ + /** @var Utopia\Database\Database $dbForInternal */ + + $usage = []; + if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { + $period = [ + '24h' => [ + 'period' => '30m', + 'limit' => 48, + ], + '7d' => [ + 'period' => '1d', + 'limit' => 7, + ], + '30d' => [ + 'period' => '1d', + 'limit' => 30, + ], + '90d' => [ + 'period' => '1d', + 'limit' => 90, + ], + ]; + + $metrics = [ + "storage.total", + "storage.files.count" + ]; + + $stats = []; + + Authorization::skip(function() use ($dbForInternal, $period, $range, $metrics, &$stats) { + foreach ($metrics as $metric) { + $requestDocs = $dbForInternal->find('stats', [ + new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]), + new Query('metric', Query::TYPE_EQUAL, [$metric]), + ], $period[$range]['limit'], 0, ['time'], [Database::ORDER_DESC]); + + $stats[$metric] = []; + foreach ($requestDocs as $requestDoc) { + $stats[$metric][] = [ + 'value' => $requestDoc->getAttribute('value'), + 'date' => $requestDoc->getAttribute('time'), + ]; + } + $stats[$metric] = array_reverse($stats[$metric]); + } + }); + + $usage = new Document([ + 'range' => $range, + 'storage' => $stats['storage.total'], + 'files' => $stats['storage.files.count'] + ]); + } + + $response->dynamic($usage, Response::MODEL_USAGE_STORAGE); + }); + +App::get('/v1/storage/:bucketId/usage') + ->desc('Get usage stats for a storage bucket') + ->groups(['api', 'storage']) + ->label('scope', 'files.read') + ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) + ->label('sdk.namespace', 'storage') + ->label('sdk.method', 'getBucketUsage') + ->label('sdk.response.code', Response::STATUS_CODE_OK) + ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) + ->label('sdk.response.model', Response::MODEL_USAGE_BUCKETS) + ->param('bucketId', '', new UID(), 'Bucket unique ID.') + ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true) + ->inject('response') + ->inject('dbForInternal') + ->action(function ($bucketId, $range, $response, $dbForInternal) { + /** @var Appwrite\Utopia\Response $response */ + /** @var Utopia\Database\Database $dbForInternal */ + + // TODO: Check if the storage bucket exists else throw 404 + + $usage = []; + if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { + $period = [ + '24h' => [ + 'period' => '30m', + 'limit' => 48, + ], + '7d' => [ + 'period' => '1d', + 'limit' => 7, + ], + '30d' => [ + 'period' => '1d', + 'limit' => 30, + ], + '90d' => [ + 'period' => '1d', + 'limit' => 90, + ], + ]; + + $metrics = [ + "storage.buckets.$bucketId.files.count", + "storage.buckets.$bucketId.files.create", + "storage.buckets.$bucketId.files.read", + "storage.buckets.$bucketId.files.update", + "storage.buckets.$bucketId.files.delete" + ]; + + $stats = []; + + Authorization::skip(function() use ($dbForInternal, $period, $range, $metrics, &$stats) { + foreach ($metrics as $metric) { + $requestDocs = $dbForInternal->find('stats', [ + new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]), + new Query('metric', Query::TYPE_EQUAL, [$metric]), + ], $period[$range]['limit'], 0, ['time'], [Database::ORDER_DESC]); + + $stats[$metric] = []; + foreach ($requestDocs as $requestDoc) { + $stats[$metric][] = [ + 'value' => $requestDoc->getAttribute('value'), + 'date' => $requestDoc->getAttribute('time'), + ]; + } + $stats[$metric] = array_reverse($stats[$metric]); + } + }); + + $usage = new Document([ + 'range' => $range, + 'files.count' => $stats["storage.buckets.$bucketId.files.count"], + 'files.create' => $stats["storage.buckets.$bucketId.files.create"], + 'files.read' => $stats["storage.buckets.$bucketId.files.read"], + 'files.update' => $stats["storage.buckets.$bucketId.files.update"], + 'files.delete' => $stats["storage.buckets.$bucketId.files.delete"] + ]); + } + + $response->dynamic($usage, Response::MODEL_USAGE_BUCKETS); }); \ No newline at end of file diff --git a/app/controllers/api/teams.php b/app/controllers/api/teams.php index 9926f2f3b..fcb5aeb5c 100644 --- a/app/controllers/api/teams.php +++ b/app/controllers/api/teams.php @@ -57,6 +57,7 @@ App::post('/v1/teams') 'name' => $name, 'sum' => ($isPrivilegedUser || $isAppUser) ? 0 : 1, 'dateCreated' => \time(), + 'search' => implode(' ', [$teamId, $name]), ])); Authorization::reset(); @@ -107,8 +108,6 @@ App::get('/v1/teams') /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ - $queries = ($search) ? [new Query('name', Query::TYPE_SEARCH, [$search])] : []; - if (!empty($after)) { $afterTeam = $dbForInternal->getDocument('teams', $after); @@ -117,6 +116,12 @@ App::get('/v1/teams') } } + $queries = []; + + if (!empty($search)) { + $queries[] = new Query('search', Query::TYPE_SEARCH, [$search]); + } + $results = $dbForInternal->find('teams', $queries, $limit, $offset, [], [$orderType], $afterTeam ?? null); $sum = $dbForInternal->count('teams', $queries, APP_LIMIT_COUNT); @@ -179,7 +184,10 @@ App::put('/v1/teams/:teamId') throw new Exception('Team not found', 404); } - $team = $dbForInternal->updateDocument('teams', $team->getId(), $team->setAttribute('name', $name)); + $team = $dbForInternal->updateDocument('teams', $team->getId(),$team + ->setAttribute('name', $name) + ->setAttribute('search', implode(' ', [$teamId, $name])) + ); $response->dynamic($team, Response::MODEL_TEAM); }); @@ -323,6 +331,7 @@ App::post('/v1/teams/:teamId/memberships') 'sessions' => [], 'tokens' => [], 'memberships' => [], + 'search' => implode(' ', [$userId, $email, $name]), ])); } catch (Duplicate $th) { throw new Exception('Account already exists', 409); diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index ec9daf980..99748d40d 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -17,7 +17,10 @@ use Utopia\Database\Exception\Duplicate; use Utopia\Database\Validator\UID; use DeviceDetector\DeviceDetector; use Appwrite\Database\Validator\CustomId; +use Utopia\Config\Config; +use Utopia\Database\Database; use Utopia\Database\Query; +use Utopia\Database\Validator\Authorization; App::post('/v1/users') ->desc('Create User') @@ -37,9 +40,11 @@ App::post('/v1/users') ->param('name', '', new Text(128), 'User name. Max length: 128 chars.', true) ->inject('response') ->inject('dbForInternal') - ->action(function ($userId, $email, $password, $name, $response, $dbForInternal) { + ->inject('usage') + ->action(function ($userId, $email, $password, $name, $response, $dbForInternal, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ + /** @var Appwrite\Stats\Stats $usage */ $email = \strtolower($email); @@ -61,11 +66,17 @@ App::post('/v1/users') 'sessions' => [], 'tokens' => [], 'memberships' => [], + 'search' => implode(' ', [$userId, $email, $name]), + 'deleted' => false ])); } catch (Duplicate $th) { throw new Exception('Account already exists', 409); } + $usage + ->setParam('users.create', 1) + ; + $response->setStatusCode(Response::STATUS_CODE_CREATED); $response->dynamic($user, Response::MODEL_USER); }); @@ -88,24 +99,35 @@ App::get('/v1/users') ->param('orderType', 'ASC', new WhiteList(['ASC', 'DESC'], true), 'Order result by ASC or DESC order.', true) ->inject('response') ->inject('dbForInternal') - ->action(function ($search, $limit, $offset, $after, $orderType, $response, $dbForInternal) { + ->inject('usage') + ->action(function ($search, $limit, $offset, $after, $orderType, $response, $dbForInternal, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ + /** @var Appwrite\Stats\Stats $usage */ if (!empty($after)) { $afterUser = $dbForInternal->getDocument('users', $after); if ($afterUser->isEmpty()) { - throw new Exception('User for after not found', 400); + throw new Exception("User '{$after}' for the 'after' value not found.", 400); } } - $results = $dbForInternal->find('users', [], $limit, $offset, [], [$orderType], $afterUser ?? null); - $sum = $dbForInternal->count('users', [], APP_LIMIT_COUNT); + $queries = [ + new Query('deleted', Query::TYPE_EQUAL, [false]) + ]; + + if (!empty($search)) { + $queries[] = new Query('search', Query::TYPE_SEARCH, [$search]); + } + + $usage + ->setParam('users.read', 1) + ; $response->dynamic(new Document([ - 'users' => $results, - 'sum' => $sum, + 'users' => $dbForInternal->find('users', $queries, $limit, $offset, [], [$orderType], $afterUser ?? null), + 'sum' => $dbForInternal->count('users', $queries, APP_LIMIT_COUNT), ]), Response::MODEL_USER_LIST); }); @@ -123,16 +145,21 @@ App::get('/v1/users/:userId') ->param('userId', '', new UID(), 'User unique ID.') ->inject('response') ->inject('dbForInternal') - ->action(function ($userId, $response, $dbForInternal) { + ->inject('usage') + ->action(function ($userId, $response, $dbForInternal, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ + /** @var Appwrite\Stats\Stats $usage */ $user = $dbForInternal->getDocument('users', $userId); - if ($user->isEmpty()) { + if ($user->isEmpty() || $user->getAttribute('deleted')) { throw new Exception('User not found', 404); } + $usage + ->setParam('users.read', 1) + ; $response->dynamic($user, Response::MODEL_USER); }); @@ -150,18 +177,23 @@ App::get('/v1/users/:userId/prefs') ->param('userId', '', new UID(), 'User unique ID.') ->inject('response') ->inject('dbForInternal') - ->action(function ($userId, $response, $dbForInternal) { + ->inject('usage') + ->action(function ($userId, $response, $dbForInternal, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ + /** @var Appwrite\Stats\Stats $usage */ $user = $dbForInternal->getDocument('users', $userId); - if ($user->isEmpty()) { + if ($user->isEmpty() || $user->getAttribute('deleted')) { throw new Exception('User not found', 404); } $prefs = $user->getAttribute('prefs', new \stdClass()); + $usage + ->setParam('users.read', 1) + ; $response->dynamic(new Document($prefs), Response::MODEL_PREFERENCES); }); @@ -180,14 +212,16 @@ App::get('/v1/users/:userId/sessions') ->inject('response') ->inject('dbForInternal') ->inject('locale') - ->action(function ($userId, $response, $dbForInternal, $locale) { + ->inject('usage') + ->action(function ($userId, $response, $dbForInternal, $locale, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ /** @var Utopia\Locale\Locale $locale */ + /** @var Appwrite\Stats\Stats $usage */ $user = $dbForInternal->getDocument('users', $userId); - if ($user->isEmpty()) { + if ($user->isEmpty() || $user->getAttribute('deleted')) { throw new Exception('User not found', 404); } @@ -203,6 +237,9 @@ App::get('/v1/users/:userId/sessions') $sessions[$key] = $session; } + $usage + ->setParam('users.read', 1) + ; $response->dynamic(new Document([ 'sessions' => $sessions, 'sum' => count($sessions), @@ -225,16 +262,18 @@ App::get('/v1/users/:userId/logs') ->inject('dbForInternal') ->inject('locale') ->inject('geodb') - ->action(function ($userId, $response, $dbForInternal, $locale, $geodb) { + ->inject('usage') + ->action(function ($userId, $response, $dbForInternal, $locale, $geodb, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Document $project */ /** @var Utopia\Database\Database $dbForInternal */ /** @var Utopia\Locale\Locale $locale */ /** @var MaxMind\Db\Reader $geodb */ + /** @var Appwrite\Stats\Stats $usage */ $user = $dbForInternal->getDocument('users', $userId); - if ($user->isEmpty()) { + if ($user->isEmpty() || $user->getAttribute('deleted')) { throw new Exception('User not found', 404); } @@ -312,6 +351,9 @@ App::get('/v1/users/:userId/logs') } } + $usage + ->setParam('users.read', 1) + ; $response->dynamic(new Document(['logs' => $output]), Response::MODEL_LOG_LIST); }); @@ -331,18 +373,23 @@ App::patch('/v1/users/:userId/status') ->param('status', null, new Boolean(true), 'User Status. To activate the user pass `true` and to block the user pass `false`') ->inject('response') ->inject('dbForInternal') - ->action(function ($userId, $status, $response, $dbForInternal) { + ->inject('usage') + ->action(function ($userId, $status, $response, $dbForInternal, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ + /** @var Appwrite\Stats\Stats $usage */ $user = $dbForInternal->getDocument('users', $userId); - if ($user->isEmpty()) { + if ($user->isEmpty() || $user->getAttribute('deleted')) { throw new Exception('User not found', 404); } $user = $dbForInternal->updateDocument('users', $user->getId(), $user->setAttribute('status', (bool) $status)); + $usage + ->setParam('users.update', 1) + ; $response->dynamic($user, Response::MODEL_USER); }); @@ -362,18 +409,23 @@ App::patch('/v1/users/:userId/verification') ->param('emailVerification', false, new Boolean(), 'User Email Verification Status.') ->inject('response') ->inject('dbForInternal') - ->action(function ($userId, $emailVerification, $response, $dbForInternal) { + ->inject('usage') + ->action(function ($userId, $emailVerification, $response, $dbForInternal, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ + /** @var Appwrite\Stats\Stats $usage */ $user = $dbForInternal->getDocument('users', $userId); - if ($user->isEmpty()) { + if ($user->isEmpty() || $user->getAttribute('deleted')) { throw new Exception('User not found', 404); } $user = $dbForInternal->updateDocument('users', $user->getId(), $user->setAttribute('emailVerification', $emailVerification)); + $usage + ->setParam('users.update', 1) + ; $response->dynamic($user, Response::MODEL_USER); }); @@ -401,7 +453,7 @@ App::patch('/v1/users/:userId/name') $user = $dbForInternal->getDocument('users', $userId); - if ($user->isEmpty()) { + if ($user->isEmpty() || $user->getAttribute('deleted')) { throw new Exception('User not found', 404); } @@ -440,7 +492,7 @@ App::patch('/v1/users/:userId/password') $user = $dbForInternal->getDocument('users', $userId); - if ($user->isEmpty()) { + if ($user->isEmpty() || $user->getAttribute('deleted')) { throw new Exception('User not found', 404); } @@ -480,7 +532,7 @@ App::patch('/v1/users/:userId/email') $user = $dbForInternal->getDocument('users', $userId); - if ($user->isEmpty()) { + if ($user->isEmpty() || $user->getAttribute('deleted')) { throw new Exception('User not found', 404); } @@ -516,18 +568,23 @@ App::patch('/v1/users/:userId/prefs') ->param('prefs', '', new Assoc(), 'Prefs key-value JSON object.') ->inject('response') ->inject('dbForInternal') - ->action(function ($userId, $prefs, $response, $dbForInternal) { + ->inject('usage') + ->action(function ($userId, $prefs, $response, $dbForInternal, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ + /** @var Appwrite\Stats\Stats $usage */ $user = $dbForInternal->getDocument('users', $userId); - if ($user->isEmpty()) { + if ($user->isEmpty() || $user->getAttribute('deleted')) { throw new Exception('User not found', 404); } $user = $dbForInternal->updateDocument('users', $user->getId(), $user->setAttribute('prefs', $prefs)); + $usage + ->setParam('users.update', 1) + ; $response->dynamic(new Document($prefs), Response::MODEL_PREFERENCES); }); @@ -547,14 +604,16 @@ App::delete('/v1/users/:userId/sessions/:sessionId') ->inject('response') ->inject('dbForInternal') ->inject('events') - ->action(function ($userId, $sessionId, $response, $dbForInternal, $events) { + ->inject('usage') + ->action(function ($userId, $sessionId, $response, $dbForInternal, $events, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ /** @var Appwrite\Event\Event $events */ + /** @var Appwrite\Stats\Stats $usage */ $user = $dbForInternal->getDocument('users', $userId); - if ($user->isEmpty()) { + if ($user->isEmpty() || $user->getAttribute('deleted')) { throw new Exception('User not found', 404); } @@ -577,6 +636,11 @@ App::delete('/v1/users/:userId/sessions/:sessionId') } } + $usage + ->setParam('users.update', 1) + ->setParam('users.sessions.delete', 1) + ; + $response->noContent(); }); @@ -595,14 +659,16 @@ App::delete('/v1/users/:userId/sessions') ->inject('response') ->inject('dbForInternal') ->inject('events') - ->action(function ($userId, $response, $dbForInternal, $events) { + ->inject('usage') + ->action(function ($userId, $response, $dbForInternal, $events, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ /** @var Appwrite\Event\Event $events */ + /** @var Appwrite\Stats\Stats $usage */ $user = $dbForInternal->getDocument('users', $userId); - if ($user->isEmpty()) { + if ($user->isEmpty() || $user->getAttribute('deleted')) { throw new Exception('User not found', 404); } @@ -618,6 +684,10 @@ App::delete('/v1/users/:userId/sessions') ->setParam('eventData', $response->output($user, Response::MODEL_USER)) ; + $usage + ->setParam('users.update', 1) + ->setParam('users.sessions.delete', 1) + ; $response->noContent(); }); @@ -637,35 +707,132 @@ App::delete('/v1/users/:userId') ->inject('dbForInternal') ->inject('events') ->inject('deletes') - ->action(function ($userId, $response, $dbForInternal, $events, $deletes) { + ->inject('usage') + ->action(function ($userId, $response, $dbForInternal, $events, $deletes, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ /** @var Appwrite\Event\Event $events */ /** @var Appwrite\Event\Event $deletes */ + /** @var Appwrite\Stats\Stats $usage */ $user = $dbForInternal->getDocument('users', $userId); - if ($user->isEmpty()) { + if ($user->isEmpty() || $user->getAttribute('deleted')) { throw new Exception('User not found', 404); } - if (!$dbForInternal->deleteDocument('users', $userId)) { - throw new Exception('Failed to remove user from DB', 500); - } - - // $dbForInternal->createDocument('users', new Document([ - // '$id' => $userId, - // '$read' => ['role:all'], - // ])); + // clone user object to send to workers + $clone = clone $user; + + $user + ->setAttribute("name", null) + ->setAttribute("email", null) + ->setAttribute("password", null) + ->setAttribute("deleted", true) + ; + + $dbForInternal->updateDocument('users', $userId, $user); $deletes ->setParam('type', DELETE_TYPE_DOCUMENT) - ->setParam('document', $user) + ->setParam('document', $clone) ; $events - ->setParam('eventData', $response->output($user, Response::MODEL_USER)) + ->setParam('eventData', $response->output($clone, Response::MODEL_USER)) + ; + + $usage + ->setParam('users.delete', 1) ; $response->noContent(); }); + +App::get('/v1/users/usage') + ->desc('Get usage stats for the users API') + ->groups(['api', 'users']) + ->label('scope', 'users.read') + ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) + ->label('sdk.namespace', 'users') + ->label('sdk.method', 'getUsage') + ->label('sdk.response.code', Response::STATUS_CODE_OK) + ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) + ->label('sdk.response.model', Response::MODEL_USAGE_USERS) + ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true) + ->param('provider', '', new WhiteList(\array_merge(['email', 'anonymous'], \array_map(function($value) { return "oauth-".$value; }, \array_keys(Config::getParam('providers', [])))), true), 'Provider Name.', true) + ->inject('response') + ->inject('dbForInternal') + ->inject('register') + ->action(function ($range, $provider, $response, $dbForInternal) { + /** @var Appwrite\Utopia\Response $response */ + /** @var Utopia\Database\Database $dbForInternal */ + + $usage = []; + if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { + $period = [ + '24h' => [ + 'period' => '30m', + 'limit' => 48, + ], + '7d' => [ + 'period' => '1d', + 'limit' => 7, + ], + '30d' => [ + 'period' => '1d', + 'limit' => 30, + ], + '90d' => [ + 'period' => '1d', + 'limit' => 90, + ], + ]; + + $metrics = [ + "users.count", + "users.create", + "users.read", + "users.update", + "users.delete", + "users.sessions.create", + "users.sessions.$provider.create", + "users.sessions.delete" + ]; + + $stats = []; + + Authorization::skip(function() use ($dbForInternal, $period, $range, $metrics, &$stats) { + foreach ($metrics as $metric) { + $requestDocs = $dbForInternal->find('stats', [ + new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]), + new Query('metric', Query::TYPE_EQUAL, [$metric]), + ], $period[$range]['limit'], 0, ['time'], [Database::ORDER_DESC]); + + $stats[$metric] = []; + foreach ($requestDocs as $requestDoc) { + $stats[$metric][] = [ + 'value' => $requestDoc->getAttribute('value'), + 'date' => $requestDoc->getAttribute('time'), + ]; + } + $stats[$metric] = array_reverse($stats[$metric]); + } + }); + + $usage = new Document([ + 'range' => $range, + 'users.count' => $stats["users.count"], + 'users.create' => $stats["users.create"], + 'users.read' => $stats["users.read"], + 'users.update' => $stats["users.update"], + 'users.delete' => $stats["users.delete"], + 'sessions.create' => $stats["users.sessions.create"], + 'sessions.provider.create' => $stats["users.sessions.$provider.create"], + 'sessions.delete' => $stats["users.sessions.delete"] + ]); + + } + + $response->dynamic($usage, Response::MODEL_USAGE_USERS); + }); \ No newline at end of file diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 3707eebfa..b7953da49 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -104,6 +104,7 @@ App::init(function ($utopia, $request, $response, $project, $user, $events, $aud ->setParam('httpRequest', 1) ->setParam('httpUrl', $request->getHostname().$request->getURI()) ->setParam('httpMethod', $request->getMethod()) + ->setParam('httpPath', $route->getPath()) ->setParam('networkRequestSize', 0) ->setParam('networkResponseSize', 0) ->setParam('storage', 0) diff --git a/app/controllers/web/home.php b/app/controllers/web/home.php index 1d9e86d87..9f4d2412a 100644 --- a/app/controllers/web/home.php +++ b/app/controllers/web/home.php @@ -387,11 +387,10 @@ App::get('/specs/:format') } $routes[] = $route; - $model = $response->getModel($route->getLabel('sdk.response.model', 'none')); - - if($model) { - $models[$model->getType()] = $model; - } + $modelLabel = $route->getLabel('sdk.response.model', 'none'); + $model = \is_array($modelLabel) ? \array_map(function($m) use($response) { + return $response->getModel($m); + }, $modelLabel) : $response->getModel($modelLabel); } } diff --git a/app/init.php b/app/init.php index 54a6428cd..0007beca7 100644 --- a/app/init.php +++ b/app/init.php @@ -17,6 +17,7 @@ ini_set('display_startup_errors', 1); ini_set('default_socket_timeout', -1); error_reporting(E_ALL); +use Appwrite\Extend\PDO; use Ahc\Jwt\JWT; use Ahc\Jwt\JWTException; use Appwrite\Auth\Auth; @@ -30,6 +31,7 @@ use Appwrite\Network\Validator\URL; use Appwrite\OpenSSL\OpenSSL; use Appwrite\Stats\Stats; use Utopia\App; +use Utopia\CLI\Console; use Utopia\View; use Utopia\Config\Config; use Utopia\Locale\Locale; @@ -95,6 +97,7 @@ const DELETE_TYPE_EXECUTIONS = 'executions'; const DELETE_TYPE_AUDIT = 'audit'; const DELETE_TYPE_ABUSE = 'abuse'; const DELETE_TYPE_CERTIFICATES = 'certificates'; +const DELETE_TYPE_USAGE = 'usage'; // Mail Worker Types const MAIL_TYPE_VERIFICATION = 'verification'; const MAIL_TYPE_RECOVERY = 'recovery'; @@ -244,7 +247,7 @@ Database::addFilter('subQueryAttributes', return $database ->find('attributes', [ new Query('collectionId', Query::TYPE_EQUAL, [$document->getId()]) - ], 100, 0, []); + ], $database->getAttributeLimit(), 0, []); } ); @@ -256,7 +259,55 @@ Database::addFilter('subQueryIndexes', return $database ->find('indexes', [ new Query('collectionId', Query::TYPE_EQUAL, [$document->getId()]) - ], 100, 0, []); + ], 64, 0, []); + } +); + +Database::addFilter('subQueryPlatforms', + function($value) { + return null; + }, + function($value, Document $document, Database $database) { + return $database + ->find('platforms', [ + new Query('projectId', Query::TYPE_EQUAL, [$document->getId()]) + ], $database->getIndexLimit(), 0, []); + } +); + +Database::addFilter('subQueryDomains', + function($value) { + return null; + }, + function($value, Document $document, Database $database) { + return $database + ->find('domains', [ + new Query('projectId', Query::TYPE_EQUAL, [$document->getId()]) + ], $database->getIndexLimit(), 0, []); + } +); + +Database::addFilter('subQueryKeys', + function($value) { + return null; + }, + function($value, Document $document, Database $database) { + return $database + ->find('keys', [ + new Query('projectId', Query::TYPE_EQUAL, [$document->getId()]) + ], $database->getIndexLimit(), 0, []); + } +); + +Database::addFilter('subQueryWebhooks', + function($value) { + return null; + }, + function($value, Document $document, Database $database) { + return $database + ->find('webhooks', [ + new Query('projectId', Query::TYPE_EQUAL, [$document->getId()]) + ], $database->getIndexLimit(), 0, []); } ); @@ -410,6 +461,29 @@ $register->set('smtp', function () { $register->set('geodb', function () { return new Reader(__DIR__.'/db/DBIP/dbip-country-lite-2021-06.mmdb'); }); +$register->set('db', function () { // This is usually for our workers or CLI commands scope + $dbHost = App::getEnv('_APP_DB_HOST', ''); + $dbUser = App::getEnv('_APP_DB_USER', ''); + $dbPass = App::getEnv('_APP_DB_PASS', ''); + $dbScheme = App::getEnv('_APP_DB_SCHEMA', ''); + + $pdo = new PDO("mysql:host={$dbHost};dbname={$dbScheme};charset=utf8mb4", $dbUser, $dbPass, array( + PDO::MYSQL_ATTR_INIT_COMMAND => 'SET NAMES utf8mb4', + PDO::ATTR_TIMEOUT => 3, // Seconds + PDO::ATTR_PERSISTENT => true, + PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC, + PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION, + )); + + return $pdo; +}); +$register->set('cache', function () { // This is usually for our workers or CLI commands scope + $redis = new Redis(); + $redis->pconnect(App::getEnv('_APP_REDIS_HOST', ''), App::getEnv('_APP_REDIS_PORT', '')); + $redis->setOption(Redis::OPT_READ_TIMEOUT, -1); + + return $redis; +}); /* * Localization diff --git a/app/tasks/maintenance.php b/app/tasks/maintenance.php index eccdf61b1..e0ef1adc0 100644 --- a/app/tasks/maintenance.php +++ b/app/tasks/maintenance.php @@ -39,17 +39,29 @@ $cli ]); } + function notifyDeleteUsageStats(int $interval30m, int $interval1d) + { + Resque::enqueue(Event::DELETE_QUEUE_NAME, Event::DELETE_CLASS_NAME, [ + 'type' => DELETE_TYPE_USAGE, + 'timestamp1d' => time() - $interval1d, + 'timestamp30m' => time() - $interval30m, + ]); + } + // # of days in seconds (1 day = 86400s) $interval = (int) App::getEnv('_APP_MAINTENANCE_INTERVAL', '86400'); $executionLogsRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_EXECUTION', '1209600'); $auditLogRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT', '1209600'); $abuseLogsRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_ABUSE', '86400'); + $usageStatsRetention30m = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_USAGE_30M', '129600');//36 hours + $usageStatsRetention1d = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_USAGE_1D', '8640000'); // 100 days - Console::loop(function() use ($interval, $executionLogsRetention, $abuseLogsRetention, $auditLogRetention){ + Console::loop(function() use ($interval, $executionLogsRetention, $abuseLogsRetention, $auditLogRetention, $usageStatsRetention30m, $usageStatsRetention1d) { $time = date('d-m-Y H:i:s', time()); Console::info("[{$time}] Notifying deletes workers every {$interval} seconds"); notifyDeleteExecutionLogs($executionLogsRetention); notifyDeleteAbuseLogs($abuseLogsRetention); notifyDeleteAuditLogs($auditLogRetention); + notifyDeleteUsageStats($usageStatsRetention30m, $usageStatsRetention1d); }, $interval); }); \ No newline at end of file diff --git a/app/tasks/usage.php b/app/tasks/usage.php new file mode 100644 index 000000000..701037fcc --- /dev/null +++ b/app/tasks/usage.php @@ -0,0 +1,586 @@ +task('usage') + ->desc('Schedules syncing data from influxdb to Appwrite console db') + ->action(function () use ($register) { + Console::title('Usage Aggregation V1'); + Console::success(APP_NAME . ' usage aggregation process v1 has started'); + + $interval = (int) App::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '30'); // 30 seconds (by default) + $periods = [ + [ + 'key' => '30m', + 'startTime' => '-24 hours', + ], + [ + 'key' => '1d', + 'startTime' => '-90 days', + ], + ]; + + // all the metrics that we are collecting at the moment + $globalMetrics = [ + 'requests' => [ + 'table' => 'appwrite_usage_requests_all', + ], + 'network' => [ + 'table' => 'appwrite_usage_network_all', + ], + 'executions' => [ + 'table' => 'appwrite_usage_executions_all', + ], + 'database.collections.create' => [ + 'table' => 'appwrite_usage_database_collections_create', + ], + 'database.collections.read' => [ + 'table' => 'appwrite_usage_database_collections_read', + ], + 'database.collections.update' => [ + 'table' => 'appwrite_usage_database_collections_update', + ], + 'database.collections.delete' => [ + 'table' => 'appwrite_usage_database_collections_delete', + ], + 'database.documents.create' => [ + 'table' => 'appwrite_usage_database_documents_create', + ], + 'database.documents.read' => [ + 'table' => 'appwrite_usage_database_documents_read', + ], + 'database.documents.update' => [ + 'table' => 'appwrite_usage_database_documents_update', + ], + 'database.documents.delete' => [ + 'table' => 'appwrite_usage_database_documents_delete', + ], + 'database.collections.collectionId.documents.create' => [ + 'table' => 'appwrite_usage_database_documents_create', + 'groupBy' => 'collectionId', + ], + 'database.collections.collectionId.documents.read' => [ + 'table' => 'appwrite_usage_database_documents_read', + 'groupBy' => 'collectionId', + ], + 'database.collections.collectionId.documents.update' => [ + 'table' => 'appwrite_usage_database_documents_update', + 'groupBy' => 'collectionId', + ], + 'database.collections.collectionId.documents.delete' => [ + 'table' => 'appwrite_usage_database_documents_delete', + 'groupBy' => 'collectionId', + ], + 'storage.buckets.bucketId.files.create' => [ + 'table' => 'appwrite_usage_storage_files_create', + 'groupBy' => 'bucketId', + ], + 'storage.buckets.bucketId.files.read' => [ + 'table' => 'appwrite_usage_storage_files_read', + 'groupBy' => 'bucketId', + ], + 'storage.buckets.bucketId.files.update' => [ + 'table' => 'appwrite_usage_storage_files_update', + 'groupBy' => 'bucketId', + ], + 'storage.buckets.bucketId.files.delete' => [ + 'table' => 'appwrite_usage_storage_files_delete', + 'groupBy' => 'bucketId', + ], + 'users.create' => [ + 'table' => 'appwrite_usage_users_create', + ], + 'users.read' => [ + 'table' => 'appwrite_usage_users_read', + ], + 'users.update' => [ + 'table' => 'appwrite_usage_users_update', + ], + 'users.delete' => [ + 'table' => 'appwrite_usage_users_delete', + ], + 'users.sessions.create' => [ + 'table' => 'appwrite_usage_users_sessions_create', + ], + 'users.sessions.provider.create' => [ + 'table' => 'appwrite_usage_users_sessions_create', + 'groupBy' => 'provider', + ], + 'users.sessions.delete' => [ + 'table' => 'appwrite_usage_users_sessions_delete', + ], + 'functions.functionId.executions' => [ + 'table' => 'appwrite_usage_executions_all', + 'groupBy' => 'functionId', + ], + 'functions.functionId.compute' => [ + 'table' => 'appwrite_usage_executions_time', + 'groupBy' => 'functionId', + ], + 'functions.functionId.failures' => [ + 'table' => 'appwrite_usage_executions_all', + 'groupBy' => 'functionId', + 'filters' => [ + 'functionStatus' => 'failed', + ], + ], + ]; + + // TODO Maybe move this to the setResource method, and reuse in the http.php file + $attempts = 0; + $max = 10; + $sleep = 1; + + do { // connect to db + try { + $attempts++; + $db = $register->get('db'); + $redis = $register->get('cache'); + break; // leave the do-while if successful + } catch (\Exception $e) { + Console::warning("Database not ready. Retrying connection ({$attempts})..."); + if ($attempts >= $max) { + throw new \Exception('Failed to connect to database: ' . $e->getMessage()); + } + sleep($sleep); + } + } while ($attempts < $max); + + // TODO use inject + $cacheAdapter = new Cache(new Redis($redis)); + $dbForProject = new Database(new MariaDB($db), $cacheAdapter); + $dbForConsole = new Database(new MariaDB($db), $cacheAdapter); + $dbForConsole->setNamespace('project_console_internal'); + + $latestTime = []; + + Authorization::disable(); + + $iterations = 0; + Console::loop(function () use ($interval, $register, $dbForProject, $dbForConsole, $globalMetrics, $periods, &$latestTime, &$iterations) { + $now = date('d-m-Y H:i:s', time()); + Console::info("[{$now}] Aggregating usage data every {$interval} seconds"); + + $loopStart = microtime(true); + + /** + * Aggregate InfluxDB every 30 seconds + * @var InfluxDB\Client $client + */ + $client = $register->get('influxdb'); + if ($client) { + $attempts = 0; + $max = 10; + $sleep = 1; + + $database = $client->selectDB('telegraf'); + do { // check if telegraf database is ready + $attempts++; + if(!in_array('telegraf', $client->listDatabases())) { + Console::warning("InfluxDB not ready. Retrying connection ({$attempts})..."); + if($attempts >= $max) { + throw new \Exception('InfluxDB database not ready yet'); + } + sleep($sleep); + } else { + break; // leave the do-while if successful + } + } while ($attempts < $max); + + // sync data + foreach ($globalMetrics as $metric => $options) { //for each metrics + foreach ($periods as $period) { // aggregate data for each period + $start = DateTime::createFromFormat('U', \strtotime($period['startTime']))->format(DateTime::RFC3339); + if (!empty($latestTime[$metric][$period['key']])) { + $start = DateTime::createFromFormat('U', $latestTime[$metric][$period['key']])->format(DateTime::RFC3339); + } + $end = DateTime::createFromFormat('U', \strtotime('now'))->format(DateTime::RFC3339); + + $table = $options['table']; //Which influxdb table to query for this metric + $groupBy = empty($options['groupBy']) ? '' : ', "' . $options['groupBy'] . '"'; //Some sub level metrics may be grouped by other tags like collectionId, bucketId, etc + + $filters = $options['filters'] ?? []; // Some metrics might have additional filters, like function's status + if (!empty($filters)) { + $filters = ' AND ' . implode(' AND ', array_map(function ($filter, $value) { + return '"' . $filter . '"=\'' . $value . '\''; + }, array_keys($filters), array_values($filters))); + } + + $result = $database->query('SELECT sum(value) AS "value" FROM "' . $table . '" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\'' . (empty($filters) ? '' : $filters) . ' GROUP BY time(' . $period['key'] . '), "projectId"' . $groupBy . ' FILL(null)'); + + $points = $result->getPoints(); + foreach ($points as $point) { + $projectId = $point['projectId']; + + if (!empty($projectId) && $projectId != 'console') { + $dbForProject->setNamespace('project_' . $projectId . '_internal'); + $metricUpdated = $metric; + + if (!empty($groupBy)) { + $groupedBy = $point[$options['groupBy']] ?? ''; + if (empty($groupedBy)) { + continue; + } + $metricUpdated = str_replace($options['groupBy'], $groupedBy, $metric); + } + + $time = \strtotime($point['time']); + $id = \md5($time . '_' . $period['key'] . '_' . $metricUpdated); //Construct unique id for each metric using time, period and metric + $value = (!empty($point['value'])) ? $point['value'] : 0; + + try { + $document = $dbForProject->getDocument('stats', $id); + if ($document->isEmpty()) { + $dbForProject->createDocument('stats', new Document([ + '$id' => $id, + 'period' => $period['key'], + 'time' => $time, + 'metric' => $metricUpdated, + 'value' => $value, + 'type' => 0, + ])); + } else { + $dbForProject->updateDocument('stats', $document->getId(), + $document->setAttribute('value', $value)); + } + $latestTime[$metric][$period['key']] = $time; + } catch (\Exception $e) { // if projects are deleted this might fail + Console::warning("Failed to save data for project {$projectId} and metric {$metricUpdated}: {$e->getMessage()}"); + } + } + } + } + } + } + + /** + * Aggregate MariaDB every 15 minutes + * Some of the queries here might contain full-table scans. + */ + if ($iterations % 30 == 0) { // Every 15 minutes aggregate number of objects in database + + $latestProject = null; + + do { // Loop over all the projects + $attempts = 0; + $max = 10; + $sleep = 1; + + do { // list projects + try { + $attempts++; + $projects = $dbForConsole->find('projects', [], 100, cursor:$latestProject); + break; // leave the do-while if successful + } catch (\Exception $e) { + Console::warning("Console DB not ready yet. Retrying ({$attempts})..."); + if ($attempts >= $max) { + throw new \Exception('Failed access console db: ' . $e->getMessage()); + } + sleep($sleep); + } + } while ($attempts < $max); + + if (empty($projects)) { + continue; + } + + $latestProject = $projects[array_key_last($projects)]; + + foreach ($projects as $project) { + $projectId = $project->getId(); + + // Get total storage + $dbForProject->setNamespace('project_' . $projectId . '_internal'); + $storageTotal = $dbForProject->sum('files', 'sizeOriginal') + $dbForProject->sum('tags', 'size'); + + $time = (int) (floor(time() / 1800) * 1800); // Time rounded to nearest 30 minutes + $id = \md5($time . '_30m_storage.total'); //Construct unique id for each metric using time, period and metric + $document = $dbForProject->getDocument('stats', $id); + if ($document->isEmpty()) { + $dbForProject->createDocument('stats', new Document([ + '$id' => $id, + 'period' => '30m', + 'time' => $time, + 'metric' => 'storage.total', + 'value' => $storageTotal, + 'type' => 1, + ])); + } else { + $dbForProject->updateDocument('stats', $document->getId(), + $document->setAttribute('value', $storageTotal)); + } + + $time = (int) (floor(time() / 86400) * 86400); // Time rounded to nearest day + $id = \md5($time . '_1d_storage.total'); //Construct unique id for each metric using time, period and metric + $document = $dbForProject->getDocument('stats', $id); + if ($document->isEmpty()) { + $dbForProject->createDocument('stats', new Document([ + '$id' => $id, + 'period' => '1d', + 'time' => $time, + 'metric' => 'storage.total', + 'value' => $storageTotal, + 'type' => 1, + ])); + } else { + $dbForProject->updateDocument('stats', $document->getId(), + $document->setAttribute('value', $storageTotal)); + } + + $collections = [ + 'users' => [ + 'namespace' => 'internal', + ], + 'collections' => [ + 'metricPrefix' => 'database', + 'namespace' => 'internal', + 'subCollections' => [ // Some collections, like collections and later buckets have child collections that need counting + 'documents' => [ + 'namespace' => 'external', + ], + ], + ], + 'files' => [ + 'metricPrefix' => 'storage', + 'namespace' => 'internal', + ], + ]; + + foreach ($collections as $collection => $options) { + try { + $dbForProject->setNamespace("project_{$projectId}_{$options['namespace']}"); + $count = $dbForProject->count($collection); + $dbForProject->setNamespace("project_{$projectId}_internal"); + $metricPrefix = $options['metricPrefix'] ?? ''; + $metric = empty($metricPrefix) ? "{$collection}.count" : "{$metricPrefix}.{$collection}.count"; + + $time = (int) (floor(time() / 1800) * 1800); // Time rounded to nearest 30 minutes + $id = \md5($time . '_30m_' . $metric); //Construct unique id for each metric using time, period and metric + $document = $dbForProject->getDocument('stats', $id); + if ($document->isEmpty()) { + $dbForProject->createDocument('stats', new Document([ + '$id' => $id, + 'time' => $time, + 'period' => '30m', + 'metric' => $metric, + 'value' => $count, + 'type' => 1, + ])); + } else { + $dbForProject->updateDocument('stats', $document->getId(), + $document->setAttribute('value', $count)); + } + + $time = (int) (floor(time() / 86400) * 86400); // Time rounded to nearest day + $id = \md5($time . '_1d_' . $metric); //Construct unique id for each metric using time, period and metric + $document = $dbForProject->getDocument('stats', $id); + if ($document->isEmpty()) { + $dbForProject->createDocument('stats', new Document([ + '$id' => $id, + 'time' => $time, + 'period' => '1d', + 'metric' => $metric, + 'value' => $count, + 'type' => 1, + ])); + } else { + $dbForProject->updateDocument('stats', $document->getId(), + $document->setAttribute('value', $count)); + } + + $subCollections = $options['subCollections'] ?? []; + + if (empty($subCollections)) { + continue; + } + + $latestParent = null; + $subCollectionCounts = []; //total project level count of sub collections + + do { // Loop over all the parent collection document for each sub collection + $dbForProject->setNamespace("project_{$projectId}_{$options['namespace']}"); + $parents = $dbForProject->find($collection, [], 100, cursor:$latestParent); // Get all the parents for the sub collections for example for documents, this will get all the collections + + if (empty($parents)) { + continue; + } + + $latestParent = $parents[array_key_last($parents)]; + + foreach ($parents as $parent) { + foreach ($subCollections as $subCollection => $subOptions) { // Sub collection counts, like database.collections.collectionId.documents.count + $dbForProject->setNamespace("project_{$projectId}_{$subOptions['namespace']}"); + $count = $dbForProject->count($parent->getId()); + + $subCollectionCounts[$subCollection] = ($subCollectionCounts[$subCollection] ?? 0) + $count; // Project level counts for sub collections like database.documents.count + + $dbForProject->setNamespace("project_{$projectId}_internal"); + + $metric = empty($metricPrefix) ? "{$collection}.{$parent->getId()}.{$subCollection}.count" : "{$metricPrefix}.{$collection}.{$parent->getId()}.{$subCollection}.count"; + $time = (int) (floor(time() / 1800) * 1800); // Time rounded to nearest 30 minutes + $id = \md5($time . '_30m_' . $metric); //Construct unique id for each metric using time, period and metric + $document = $dbForProject->getDocument('stats', $id); + if ($document->isEmpty()) { + $dbForProject->createDocument('stats', new Document([ + '$id' => $id, + 'time' => $time, + 'period' => '30m', + 'metric' => $metric, + 'value' => $count, + 'type' => 1, + ])); + } else { + $dbForProject->updateDocument('stats', $document->getId(), + $document->setAttribute('value', $count)); + } + + $time = (int) (floor(time() / 86400) * 86400); // Time rounded to nearest day + $id = \md5($time . '_1d_' . $metric); //Construct unique id for each metric using time, period and metric + $document = $dbForProject->getDocument('stats', $id); + if ($document->isEmpty()) { + $dbForProject->createDocument('stats', new Document([ + '$id' => $id, + 'time' => $time, + 'period' => '1d', + 'metric' => $metric, + 'value' => $count, + 'type' => 1, + ])); + } else { + $dbForProject->updateDocument('stats', $document->getId(), + $document->setAttribute('value', $count)); + } + } + } + } while (!empty($parents)); + + /** + * Inserting project level counts for sub collections like database.documents.count + */ + foreach ($subCollectionCounts as $subCollection => $count) { + $dbForProject->setNamespace("project_{$projectId}_internal"); + + $metric = empty($metricPrefix) ? "{$subCollection}.count" : "{$metricPrefix}.{$subCollection}.count"; + + $time = (int) (floor(time() / 1800) * 1800); // Time rounded to nearest 30 minutes + $id = \md5($time . '_30m_' . $metric); //Construct unique id for each metric using time, period and metric + $document = $dbForProject->getDocument('stats', $id); + if ($document->isEmpty()) { + $dbForProject->createDocument('stats', new Document([ + '$id' => $id, + 'time' => $time, + 'period' => '30m', + 'metric' => $metric, + 'value' => $count, + 'type' => 1, + ])); + } else { + $dbForProject->updateDocument('stats', $document->getId(), + $document->setAttribute('value', $count)); + } + + $time = (int) (floor(time() / 86400) * 86400); // Time rounded to nearest day + $id = \md5($time . '_1d_' . $metric); //Construct unique id for each metric using time, period and metric + $document = $dbForProject->getDocument('stats', $id); + if ($document->isEmpty()) { + $dbForProject->createDocument('stats', new Document([ + '$id' => $id, + 'time' => $time, + 'period' => '1d', + 'metric' => $metric, + 'value' => $count, + 'type' => 1, + ])); + } else { + $dbForProject->updateDocument('stats', $document->getId(), + $document->setAttribute('value', $count)); + } + } + } catch (\Exception$e) { + Console::warning("Failed to save database counters data for project {$collection}: {$e->getMessage()}"); + } + } + } + } while (!empty($projects)); + } + + $iterations++; + $loopTook = microtime(true) - $loopStart; + $now = date('d-m-Y H:i:s', time()); + + Console::info("[{$now}] Aggregation took {$loopTook} seconds"); + }, $interval); + }); diff --git a/app/views/install/compose.phtml b/app/views/install/compose.phtml index 14b5a0818..28b314b18 100644 --- a/app/views/install/compose.phtml +++ b/app/views/install/compose.phtml @@ -294,6 +294,26 @@ services: - _APP_MAINTENANCE_RETENTION_ABUSE - _APP_MAINTENANCE_RETENTION_AUDIT + appwrite-usage: + image: /: + entrypoint: usage + container_name: appwrite-usage + restart: unless-stopped + networks: + - appwrite + depends_on: + - influxdb + - mariadb + environment: + - _APP_ENV + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS + - _APP_INFLUXDB_HOST + - _APP_INFLUXDB_PORT + - _APP_USAGE_AGGREGATION_INTERVAL appwrite-schedule: image: /: diff --git a/app/workers.php b/app/workers.php deleted file mode 100644 index 7dd48eaff..000000000 --- a/app/workers.php +++ /dev/null @@ -1,34 +0,0 @@ -set('db', function () { - $dbHost = App::getEnv('_APP_DB_HOST', ''); - $dbUser = App::getEnv('_APP_DB_USER', ''); - $dbPass = App::getEnv('_APP_DB_PASS', ''); - $dbScheme = App::getEnv('_APP_DB_SCHEMA', ''); - - $pdo = new PDO("mysql:host={$dbHost};dbname={$dbScheme};charset=utf8mb4", $dbUser, $dbPass, array( - PDO::MYSQL_ATTR_INIT_COMMAND => 'SET NAMES utf8mb4', - PDO::ATTR_TIMEOUT => 3, // Seconds - PDO::ATTR_PERSISTENT => true, - PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC, - PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION, - )); - - return $pdo; -}); - -$register->set('cache', function () { // Register cache connection - $redis = new Redis(); - $redis->pconnect(App::getEnv('_APP_REDIS_HOST', ''), App::getEnv('_APP_REDIS_PORT', '')); - $redis->setOption(Redis::OPT_READ_TIMEOUT, -1); - - return $redis; -}); - diff --git a/app/workers/audits.php b/app/workers/audits.php index b18a57ffd..e18b9d532 100644 --- a/app/workers/audits.php +++ b/app/workers/audits.php @@ -4,7 +4,7 @@ use Appwrite\Resque\Worker; use Utopia\Audit\Audit; use Utopia\CLI\Console; -require_once __DIR__.'/../workers.php'; +require_once __DIR__.'/../init.php'; Console::title('Audits V1 Worker'); Console::success(APP_NAME.' audits worker v1 has started'); diff --git a/app/workers/certificates.php b/app/workers/certificates.php index 96148b9e7..0b64c087f 100644 --- a/app/workers/certificates.php +++ b/app/workers/certificates.php @@ -9,7 +9,7 @@ use Utopia\Database\Query; use Utopia\Database\Validator\Authorization; use Utopia\Domains\Domain; -require_once __DIR__.'/../workers.php'; +require_once __DIR__.'/../init.php'; Console::title('Certificates V1 Worker'); Console::success(APP_NAME.' certificates worker v1 has started'); diff --git a/app/workers/database.php b/app/workers/database.php index fd3824615..8ffe3ffc3 100644 --- a/app/workers/database.php +++ b/app/workers/database.php @@ -5,7 +5,7 @@ use Utopia\CLI\Console; use Utopia\Database\Document; use Utopia\Database\Validator\Authorization; -require_once __DIR__.'/../workers.php'; +require_once __DIR__.'/../init.php'; Console::title('Database V1 Worker'); Console::success(APP_NAME.' database worker v1 has started'."\n"); @@ -94,7 +94,7 @@ class DatabaseV1 extends Worker $dbForInternal->updateDocument('attributes', $attribute->getId(), $attribute->setAttribute('status', 'failed')); } - $dbForInternal->purgeDocument('collections', $collectionId); + $dbForInternal->deleteCachedDocument('collections', $collectionId); } /** @@ -120,7 +120,58 @@ class DatabaseV1 extends Worker $dbForInternal->updateDocument('attributes', $attribute->getId(), $attribute->setAttribute('status', 'failed')); } - $dbForInternal->purgeDocument('collections', $collectionId); + // The underlying database removes/rebuilds indexes when attribute is removed + // Update indexes table with changes + /** @var Document[] $indexes */ + $indexes = $collection->getAttribute('indexes', []); + + foreach ($indexes as $index) { + /** @var string[] $attributes */ + $attributes = $index->getAttribute('attributes'); + $lengths = $index->getAttribute('lengths'); + $orders = $index->getAttribute('orders'); + + $found = \array_search($key, $attributes); + + if ($found !== false) { + // If found, remove entry from attributes, lengths, and orders + // array_values wraps array_diff to reindex array keys + // when found attribute is removed from array + $attributes = \array_values(\array_diff($attributes, [$attributes[$found]])); + $lengths = \array_values(\array_diff($lengths, [$lengths[$found]])); + $orders = \array_values(\array_diff($orders, [$orders[$found]])); + + if (empty($attributes)) { + $dbForInternal->deleteDocument('indexes', $index->getId()); + } else { + $index + ->setAttribute('attributes', $attributes, Document::SET_TYPE_ASSIGN) + ->setAttribute('lengths', $lengths, Document::SET_TYPE_ASSIGN) + ->setAttribute('orders', $orders, Document::SET_TYPE_ASSIGN) + ; + + // Check if an index exists with the same attributes and orders + $exists = false; + foreach ($indexes as $existing) { + if ($existing->getAttribute('key') !== $index->getAttribute('key') // Ignore itself + && $existing->getAttribute('attributes') === $index->getAttribute('attributes') + && $existing->getAttribute('orders') === $index->getAttribute('orders') + ) { + $exists = true; + break; + } + } + + if ($exists) { // Delete the duplicate if created, else update in db + $this->deleteIndex($collection, $index, $projectId); + } else { + $dbForInternal->updateDocument('indexes', $index->getId(), $index); + } + } + } + } + + $dbForInternal->deleteCachedDocument('collections', $collectionId); } /** @@ -150,7 +201,7 @@ class DatabaseV1 extends Worker $dbForInternal->updateDocument('indexes', $index->getId(), $index->setAttribute('status', 'failed')); } - $dbForInternal->purgeDocument('collections', $collectionId); + $dbForInternal->deleteCachedDocument('collections', $collectionId); } /** @@ -168,7 +219,7 @@ class DatabaseV1 extends Worker try { if(!$dbForExternal->deleteIndex($collectionId, $key)) { - throw new Exception('Failed to delete Attribute'); + throw new Exception('Failed to delete index'); } $dbForInternal->deleteDocument('indexes', $index->getId()); @@ -177,6 +228,6 @@ class DatabaseV1 extends Worker $dbForInternal->updateDocument('indexes', $index->getId(), $index->setAttribute('status', 'failed')); } - $dbForInternal->purgeDocument('collections', $collectionId); + $dbForInternal->deleteCachedDocument('collections', $collectionId); } } diff --git a/app/workers/deletes.php b/app/workers/deletes.php index 1894aaf76..5b1072c13 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -11,7 +11,7 @@ use Utopia\Abuse\Adapters\TimeLimit; use Utopia\CLI\Console; use Utopia\Audit\Audit; -require_once __DIR__.'/../workers.php'; +require_once __DIR__.'/../init.php'; Console::title('Deletes V1 Worker'); Console::success(APP_NAME.' deletes worker v1 has started'."\n"); @@ -44,6 +44,9 @@ class DeletesV1 extends Worker switch ($document->getCollection()) { // TODO@kodumbeats define these as constants somewhere + case 'collections': + $this->deleteCollection($document, $projectId); + break; case 'projects': $this->deleteProject($document); break; @@ -78,7 +81,10 @@ class DeletesV1 extends Worker $document = new Document($this->args['document']); $this->deleteCertificates($document); break; - + + case DELETE_TYPE_USAGE: + $this->deleteUsageStats($this->args['timestamp1d'], $this->args['timestamp30m']); + break; default: Console::error('No delete operation for type: '.$type); break; @@ -88,6 +94,51 @@ class DeletesV1 extends Worker public function shutdown(): void { } + + /** + * @param Document $document teams document + * @param string $projectId + */ + protected function deleteCollection(Document $document, string $projectId): void + { + $collectionId = $document->getId(); + + $dbForInternal = $this->getInternalDB($projectId); + $dbForExternal = $this->getExternalDB($projectId); + + $this->deleteByGroup('attributes', [ + new Query('collectionId', Query::TYPE_EQUAL, [$collectionId]) + ], $dbForInternal); + + $this->deleteByGroup('indexes', [ + new Query('collectionId', Query::TYPE_EQUAL, [$collectionId]) + ], $dbForInternal); + + $dbForExternal->deleteCollection($collectionId); + } + + /** + * @param int $timestamp1d + * @param int $timestamp30m + */ + protected function deleteUsageStats(int $timestamp1d, int $timestamp30m) { + $this->deleteForProjectIds(function($projectId) use ($timestamp1d, $timestamp30m) { + if (!($dbForInternal = $this->getInternalDB($projectId))) { + throw new Exception('Failed to get projectDB for project '.$projectId); + } + + // Delete Usage stats + $this->deleteByGroup('stats', [ + new Query('time', Query::TYPE_LESSER, [$timestamp1d]), + new Query('period', Query::TYPE_EQUAL, ['1d']), + ], $dbForInternal); + + $this->deleteByGroup('stats', [ + new Query('time', Query::TYPE_LESSER, [$timestamp30m]), + new Query('period', Query::TYPE_EQUAL, ['30m']), + ], $dbForInternal); + }); + } /** * @param Document $document teams document diff --git a/app/workers/functions.php b/app/workers/functions.php index bed0bda13..49b42b949 100644 --- a/app/workers/functions.php +++ b/app/workers/functions.php @@ -13,7 +13,7 @@ use Utopia\Database\Database; use Utopia\Database\Document; use Utopia\Database\Validator\Authorization; -require_once __DIR__.'/../workers.php'; +require_once __DIR__.'/../init.php'; Runtime::enableCoroutine(0); diff --git a/app/workers/mails.php b/app/workers/mails.php index 25abe54aa..9071c0090 100644 --- a/app/workers/mails.php +++ b/app/workers/mails.php @@ -6,7 +6,7 @@ use Utopia\App; use Utopia\CLI\Console; use Utopia\Locale\Locale; -require_once __DIR__ . '/../workers.php'; +require_once __DIR__ . '/../init.php'; Console::title('Mails V1 Worker'); Console::success(APP_NAME . ' mails worker v1 has started' . "\n"); diff --git a/app/workers/webhooks.php b/app/workers/webhooks.php index 00307bdda..a73dbb6c7 100644 --- a/app/workers/webhooks.php +++ b/app/workers/webhooks.php @@ -4,7 +4,7 @@ use Appwrite\Resque\Worker; use Utopia\App; use Utopia\CLI\Console; -require_once __DIR__.'/../workers.php'; +require_once __DIR__.'/../init.php'; Console::title('Webhooks V1 Worker'); Console::success(APP_NAME.' webhooks worker v1 has started'); diff --git a/bin/usage b/bin/usage new file mode 100755 index 000000000..2709200ae --- /dev/null +++ b/bin/usage @@ -0,0 +1,3 @@ +#!/bin/sh + +php /usr/src/code/app/cli.php usage $@ \ No newline at end of file diff --git a/composer.json b/composer.json index 32915e1ba..b3d21a4b0 100644 --- a/composer.json +++ b/composer.json @@ -45,7 +45,7 @@ "utopia-php/cache": "0.4.*", "utopia-php/cli": "0.11.*", "utopia-php/config": "0.2.*", - "utopia-php/database": "dev-feat-adjust-encodeAttribute as 0.10.0", + "utopia-php/database": "0.10.0", "utopia-php/locale": "0.4.*", "utopia-php/registry": "0.5.*", "utopia-php/preloader": "0.2.*", @@ -63,12 +63,7 @@ "adhocore/jwt": "1.1.2", "slickdeals/statsd": "3.1.0" }, - "repositories": [ - { - "type": "git", - "url": "https://github.com/utopia-php/database" - } - ], + "repositories": [], "require-dev": { "appwrite/sdk-generator": "0.13.0", "swoole/ide-helper": "4.6.7", diff --git a/composer.lock b/composer.lock index c867efc33..7eee04c86 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "31670d6cc60a22007b4ed59a7dc9448f", + "content-hash": "dfb8fa19daa736b3687617c98f309983", "packages": [ { "name": "adhocore/jwt", @@ -248,16 +248,16 @@ }, { "name": "chillerlan/php-settings-container", - "version": "2.1.1", + "version": "2.1.2", "source": { "type": "git", "url": "https://github.com/chillerlan/php-settings-container.git", - "reference": "98ccc1b31b31a53bcb563465c4961879b2b93096" + "reference": "ec834493a88682dd69652a1eeaf462789ed0c5f5" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/chillerlan/php-settings-container/zipball/98ccc1b31b31a53bcb563465c4961879b2b93096", - "reference": "98ccc1b31b31a53bcb563465c4961879b2b93096", + "url": "https://api.github.com/repos/chillerlan/php-settings-container/zipball/ec834493a88682dd69652a1eeaf462789ed0c5f5", + "reference": "ec834493a88682dd69652a1eeaf462789ed0c5f5", "shasum": "" }, "require": { @@ -307,7 +307,7 @@ "type": "ko_fi" } ], - "time": "2021-01-06T15:57:03+00:00" + "time": "2021-09-06T15:17:01+00:00" }, { "name": "colinmollenhour/credis", @@ -355,16 +355,16 @@ }, { "name": "composer/package-versions-deprecated", - "version": "1.11.99.3", + "version": "1.11.99.4", "source": { "type": "git", "url": "https://github.com/composer/package-versions-deprecated.git", - "reference": "fff576ac850c045158a250e7e27666e146e78d18" + "reference": "b174585d1fe49ceed21928a945138948cb394600" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/composer/package-versions-deprecated/zipball/fff576ac850c045158a250e7e27666e146e78d18", - "reference": "fff576ac850c045158a250e7e27666e146e78d18", + "url": "https://api.github.com/repos/composer/package-versions-deprecated/zipball/b174585d1fe49ceed21928a945138948cb394600", + "reference": "b174585d1fe49ceed21928a945138948cb394600", "shasum": "" }, "require": { @@ -408,7 +408,7 @@ "description": "Composer plugin that provides efficient querying for installed package versions (no runtime IO)", "support": { "issues": "https://github.com/composer/package-versions-deprecated/issues", - "source": "https://github.com/composer/package-versions-deprecated/tree/1.11.99.3" + "source": "https://github.com/composer/package-versions-deprecated/tree/1.11.99.4" }, "funding": [ { @@ -424,7 +424,7 @@ "type": "tidelift" } ], - "time": "2021-08-17T13:49:14+00:00" + "time": "2021-09-13T08:41:34+00:00" }, { "name": "dragonmantank/cron-expression", @@ -1984,11 +1984,17 @@ }, { "name": "utopia-php/database", - "version": "dev-feat-adjust-encodeAttribute", + "version": "0.10.0", "source": { "type": "git", - "url": "https://github.com/utopia-php/database", - "reference": "5ef32ec85143daf78796e7826453244d24f8a86a" + "url": "https://github.com/utopia-php/database.git", + "reference": "b7c60b0ec769a9050dd2b939b78ff1f5d4fa27e8" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/utopia-php/database/zipball/b7c60b0ec769a9050dd2b939b78ff1f5d4fa27e8", + "reference": "b7c60b0ec769a9050dd2b939b78ff1f5d4fa27e8", + "shasum": "" }, "require": { "ext-mongodb": "*", @@ -2011,11 +2017,7 @@ "Utopia\\Database\\": "src/Database" } }, - "autoload-dev": { - "psr-4": { - "Utopia\\Tests\\": "tests/Database" - } - }, + "notification-url": "https://packagist.org/downloads/", "license": [ "MIT" ], @@ -2037,7 +2039,11 @@ "upf", "utopia" ], - "time": "2021-08-27T20:39:51+00:00" + "support": { + "issues": "https://github.com/utopia-php/database/issues", + "source": "https://github.com/utopia-php/database/tree/0.10.0" + }, + "time": "2021-10-04T17:23:25+00:00" }, { "name": "utopia-php/domains", @@ -2576,16 +2582,16 @@ "packages-dev": [ { "name": "amphp/amp", - "version": "v2.6.0", + "version": "v2.6.1", "source": { "type": "git", "url": "https://github.com/amphp/amp.git", - "reference": "caa95edeb1ca1bf7532e9118ede4a3c3126408cc" + "reference": "c5fc66a78ee38d7ac9195a37bacaf940eb3f65ae" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/amphp/amp/zipball/caa95edeb1ca1bf7532e9118ede4a3c3126408cc", - "reference": "caa95edeb1ca1bf7532e9118ede4a3c3126408cc", + "url": "https://api.github.com/repos/amphp/amp/zipball/c5fc66a78ee38d7ac9195a37bacaf940eb3f65ae", + "reference": "c5fc66a78ee38d7ac9195a37bacaf940eb3f65ae", "shasum": "" }, "require": { @@ -2653,7 +2659,7 @@ "support": { "irc": "irc://irc.freenode.org/amphp", "issues": "https://github.com/amphp/amp/issues", - "source": "https://github.com/amphp/amp/tree/v2.6.0" + "source": "https://github.com/amphp/amp/tree/v2.6.1" }, "funding": [ { @@ -2661,7 +2667,7 @@ "type": "github" } ], - "time": "2021-07-16T20:06:06+00:00" + "time": "2021-09-23T18:43:08+00:00" }, { "name": "amphp/byte-stream", @@ -3383,16 +3389,16 @@ }, { "name": "nikic/php-parser", - "version": "v4.12.0", + "version": "v4.13.0", "source": { "type": "git", "url": "https://github.com/nikic/PHP-Parser.git", - "reference": "6608f01670c3cc5079e18c1dab1104e002579143" + "reference": "50953a2691a922aa1769461637869a0a2faa3f53" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/6608f01670c3cc5079e18c1dab1104e002579143", - "reference": "6608f01670c3cc5079e18c1dab1104e002579143", + "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/50953a2691a922aa1769461637869a0a2faa3f53", + "reference": "50953a2691a922aa1769461637869a0a2faa3f53", "shasum": "" }, "require": { @@ -3433,9 +3439,9 @@ ], "support": { "issues": "https://github.com/nikic/PHP-Parser/issues", - "source": "https://github.com/nikic/PHP-Parser/tree/v4.12.0" + "source": "https://github.com/nikic/PHP-Parser/tree/v4.13.0" }, - "time": "2021-07-21T10:44:31+00:00" + "time": "2021-09-20T12:20:58+00:00" }, { "name": "openlss/lib-array2xml", @@ -3712,16 +3718,16 @@ }, { "name": "phpdocumentor/type-resolver", - "version": "1.4.0", + "version": "1.5.1", "source": { "type": "git", "url": "https://github.com/phpDocumentor/TypeResolver.git", - "reference": "6a467b8989322d92aa1c8bf2bebcc6e5c2ba55c0" + "reference": "a12f7e301eb7258bb68acd89d4aefa05c2906cae" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpDocumentor/TypeResolver/zipball/6a467b8989322d92aa1c8bf2bebcc6e5c2ba55c0", - "reference": "6a467b8989322d92aa1c8bf2bebcc6e5c2ba55c0", + "url": "https://api.github.com/repos/phpDocumentor/TypeResolver/zipball/a12f7e301eb7258bb68acd89d4aefa05c2906cae", + "reference": "a12f7e301eb7258bb68acd89d4aefa05c2906cae", "shasum": "" }, "require": { @@ -3729,7 +3735,8 @@ "phpdocumentor/reflection-common": "^2.0" }, "require-dev": { - "ext-tokenizer": "*" + "ext-tokenizer": "*", + "psalm/phar": "^4.8" }, "type": "library", "extra": { @@ -3755,39 +3762,39 @@ "description": "A PSR-5 based resolver of Class names, Types and Structural Element Names", "support": { "issues": "https://github.com/phpDocumentor/TypeResolver/issues", - "source": "https://github.com/phpDocumentor/TypeResolver/tree/1.4.0" + "source": "https://github.com/phpDocumentor/TypeResolver/tree/1.5.1" }, - "time": "2020-09-17T18:55:26+00:00" + "time": "2021-10-02T14:08:47+00:00" }, { "name": "phpspec/prophecy", - "version": "1.13.0", + "version": "1.14.0", "source": { "type": "git", "url": "https://github.com/phpspec/prophecy.git", - "reference": "be1996ed8adc35c3fd795488a653f4b518be70ea" + "reference": "d86dfc2e2a3cd366cee475e52c6bb3bbc371aa0e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpspec/prophecy/zipball/be1996ed8adc35c3fd795488a653f4b518be70ea", - "reference": "be1996ed8adc35c3fd795488a653f4b518be70ea", + "url": "https://api.github.com/repos/phpspec/prophecy/zipball/d86dfc2e2a3cd366cee475e52c6bb3bbc371aa0e", + "reference": "d86dfc2e2a3cd366cee475e52c6bb3bbc371aa0e", "shasum": "" }, "require": { "doctrine/instantiator": "^1.2", - "php": "^7.2 || ~8.0, <8.1", + "php": "^7.2 || ~8.0, <8.2", "phpdocumentor/reflection-docblock": "^5.2", "sebastian/comparator": "^3.0 || ^4.0", "sebastian/recursion-context": "^3.0 || ^4.0" }, "require-dev": { - "phpspec/phpspec": "^6.0", + "phpspec/phpspec": "^6.0 || ^7.0", "phpunit/phpunit": "^8.0 || ^9.0" }, "type": "library", "extra": { "branch-alias": { - "dev-master": "1.11.x-dev" + "dev-master": "1.x-dev" } }, "autoload": { @@ -3822,29 +3829,29 @@ ], "support": { "issues": "https://github.com/phpspec/prophecy/issues", - "source": "https://github.com/phpspec/prophecy/tree/1.13.0" + "source": "https://github.com/phpspec/prophecy/tree/1.14.0" }, - "time": "2021-03-17T13:42:18+00:00" + "time": "2021-09-10T09:02:12+00:00" }, { "name": "phpunit/php-code-coverage", - "version": "9.2.6", + "version": "9.2.7", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/php-code-coverage.git", - "reference": "f6293e1b30a2354e8428e004689671b83871edde" + "reference": "d4c798ed8d51506800b441f7a13ecb0f76f12218" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/f6293e1b30a2354e8428e004689671b83871edde", - "reference": "f6293e1b30a2354e8428e004689671b83871edde", + "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/d4c798ed8d51506800b441f7a13ecb0f76f12218", + "reference": "d4c798ed8d51506800b441f7a13ecb0f76f12218", "shasum": "" }, "require": { "ext-dom": "*", "ext-libxml": "*", "ext-xmlwriter": "*", - "nikic/php-parser": "^4.10.2", + "nikic/php-parser": "^4.12.0", "php": ">=7.3", "phpunit/php-file-iterator": "^3.0.3", "phpunit/php-text-template": "^2.0.2", @@ -3893,7 +3900,7 @@ ], "support": { "issues": "https://github.com/sebastianbergmann/php-code-coverage/issues", - "source": "https://github.com/sebastianbergmann/php-code-coverage/tree/9.2.6" + "source": "https://github.com/sebastianbergmann/php-code-coverage/tree/9.2.7" }, "funding": [ { @@ -3901,7 +3908,7 @@ "type": "github" } ], - "time": "2021-03-28T07:26:59+00:00" + "time": "2021-09-17T05:39:03+00:00" }, { "name": "phpunit/php-file-iterator", @@ -5313,16 +5320,16 @@ }, { "name": "symfony/console", - "version": "v5.3.6", + "version": "v5.3.7", "source": { "type": "git", "url": "https://github.com/symfony/console.git", - "reference": "51b71afd6d2dc8f5063199357b9880cea8d8bfe2" + "reference": "8b1008344647462ae6ec57559da166c2bfa5e16a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/console/zipball/51b71afd6d2dc8f5063199357b9880cea8d8bfe2", - "reference": "51b71afd6d2dc8f5063199357b9880cea8d8bfe2", + "url": "https://api.github.com/repos/symfony/console/zipball/8b1008344647462ae6ec57559da166c2bfa5e16a", + "reference": "8b1008344647462ae6ec57559da166c2bfa5e16a", "shasum": "" }, "require": { @@ -5392,7 +5399,7 @@ "terminal" ], "support": { - "source": "https://github.com/symfony/console/tree/v5.3.6" + "source": "https://github.com/symfony/console/tree/v5.3.7" }, "funding": [ { @@ -5408,7 +5415,7 @@ "type": "tidelift" } ], - "time": "2021-07-27T19:10:22+00:00" + "time": "2021-08-25T20:02:16+00:00" }, { "name": "symfony/deprecation-contracts", @@ -5882,16 +5889,16 @@ }, { "name": "symfony/string", - "version": "v5.3.3", + "version": "v5.3.7", "source": { "type": "git", "url": "https://github.com/symfony/string.git", - "reference": "bd53358e3eccec6a670b5f33ab680d8dbe1d4ae1" + "reference": "8d224396e28d30f81969f083a58763b8b9ceb0a5" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/string/zipball/bd53358e3eccec6a670b5f33ab680d8dbe1d4ae1", - "reference": "bd53358e3eccec6a670b5f33ab680d8dbe1d4ae1", + "url": "https://api.github.com/repos/symfony/string/zipball/8d224396e28d30f81969f083a58763b8b9ceb0a5", + "reference": "8d224396e28d30f81969f083a58763b8b9ceb0a5", "shasum": "" }, "require": { @@ -5945,7 +5952,7 @@ "utf8" ], "support": { - "source": "https://github.com/symfony/string/tree/v5.3.3" + "source": "https://github.com/symfony/string/tree/v5.3.7" }, "funding": [ { @@ -5961,7 +5968,7 @@ "type": "tidelift" } ], - "time": "2021-06-27T11:44:38+00:00" + "time": "2021-08-26T08:00:08+00:00" }, { "name": "theseer/tokenizer", @@ -6015,16 +6022,16 @@ }, { "name": "twig/twig", - "version": "v2.14.6", + "version": "v2.14.7", "source": { "type": "git", "url": "https://github.com/twigphp/Twig.git", - "reference": "27e5cf2b05e3744accf39d4c68a3235d9966d260" + "reference": "8e202327ee1ed863629de9b18a5ec70ac614d88f" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/twigphp/Twig/zipball/27e5cf2b05e3744accf39d4c68a3235d9966d260", - "reference": "27e5cf2b05e3744accf39d4c68a3235d9966d260", + "url": "https://api.github.com/repos/twigphp/Twig/zipball/8e202327ee1ed863629de9b18a5ec70ac614d88f", + "reference": "8e202327ee1ed863629de9b18a5ec70ac614d88f", "shasum": "" }, "require": { @@ -6034,7 +6041,7 @@ }, "require-dev": { "psr/container": "^1.0", - "symfony/phpunit-bridge": "^4.4.9|^5.0.9" + "symfony/phpunit-bridge": "^4.4.9|^5.0.9|^6.0" }, "type": "library", "extra": { @@ -6078,7 +6085,7 @@ ], "support": { "issues": "https://github.com/twigphp/Twig/issues", - "source": "https://github.com/twigphp/Twig/tree/v2.14.6" + "source": "https://github.com/twigphp/Twig/tree/v2.14.7" }, "funding": [ { @@ -6090,7 +6097,7 @@ "type": "tidelift" } ], - "time": "2021-05-16T12:12:47+00:00" + "time": "2021-09-17T08:39:54+00:00" }, { "name": "vimeo/psalm", @@ -6248,18 +6255,9 @@ "time": "2015-12-17T08:42:14+00:00" } ], - "aliases": [ - { - "package": "utopia-php/database", - "version": "dev-feat-adjust-encodeAttribute", - "alias": "0.10.0", - "alias_normalized": "0.10.0.0" - } - ], + "aliases": [], "minimum-stability": "stable", - "stability-flags": { - "utopia-php/database": 20 - }, + "stability-flags": [], "prefer-stable": false, "prefer-lowest": false, "platform": { diff --git a/docker-compose.yml b/docker-compose.yml index ac12f2010..ade61dc1f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -63,7 +63,7 @@ services: - ./psalm.xml:/usr/src/code/psalm.xml - ./tests:/usr/src/code/tests - ./app:/usr/src/code/app - # - ./vendor/utopia-php/database:/usr/src/code/vendor/utopia-php/database + # - ./vendor:/usr/src/code/vendor - ./docs:/usr/src/code/docs - ./public:/usr/src/code/public - ./src:/usr/src/code/src @@ -73,7 +73,6 @@ services: - mariadb - redis # - clamav - - influxdb entrypoint: - php - -e @@ -112,8 +111,6 @@ services: - _APP_SMTP_USERNAME - _APP_SMTP_PASSWORD - _APP_USAGE_STATS - - _APP_INFLUXDB_HOST - - _APP_INFLUXDB_PORT - _APP_STORAGE_LIMIT - _APP_FUNCTIONS_TIMEOUT - _APP_FUNCTIONS_CONTAINERS @@ -349,6 +346,37 @@ services: - _APP_MAINTENANCE_RETENTION_ABUSE - _APP_MAINTENANCE_RETENTION_AUDIT + appwrite-usage: + entrypoint: + - php + - -e + - /usr/src/code/app/cli.php + - usage + container_name: appwrite-usage + build: + context: . + args: + - DEBUG=false + networks: + - appwrite + volumes: + - ./app:/usr/src/code/app + - ./src:/usr/src/code/src + - ./dev:/usr/local/dev + depends_on: + - influxdb + - mariadb + environment: + - _APP_ENV + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS + - _APP_INFLUXDB_HOST + - _APP_INFLUXDB_PORT + - _APP_USAGE_SYNC_INTERVAL + appwrite-schedule: entrypoint: schedule container_name: appwrite-schedule diff --git a/src/Appwrite/Database/Database.php b/src/Appwrite/Database/Database.php index e6cfa001d..37978ab65 100644 --- a/src/Appwrite/Database/Database.php +++ b/src/Appwrite/Database/Database.php @@ -577,7 +577,7 @@ class Database { if (!isset(self::$filters[$name])) { return $value; - throw new Exception('Filter not found'); + throw new Exception("Filter '{$name}' not found"); } try { @@ -599,7 +599,7 @@ class Database { if (!isset(self::$filters[$name])) { return $value; - throw new Exception('Filter not found'); + throw new Exception("Filter '{$name}' not found"); } try { diff --git a/src/Appwrite/Specification/Format/OpenAPI3.php b/src/Appwrite/Specification/Format/OpenAPI3.php index 353369eb6..863a7a962 100644 --- a/src/Appwrite/Specification/Format/OpenAPI3.php +++ b/src/Appwrite/Specification/Format/OpenAPI3.php @@ -4,7 +4,6 @@ namespace Appwrite\Specification\Format; use Appwrite\Specification\Format; use Appwrite\Template\Template; -use stdClass; use Utopia\Validator; class OpenAPI3 extends Format @@ -21,6 +20,34 @@ class OpenAPI3 extends Format return 'Open API 3'; } + /** + * Get Used Models + * + * Recursively get all used models + * + * @param object $model + * @param array $models + * + * @return void + */ + protected function getUsedModels($model, array &$usedModels) + { + if (is_string($model) && !in_array($model, ['string', 'integer', 'boolean', 'json', 'float', 'double'])) { + $usedModels[] = $model; + return; + } + if (!is_object($model)) return; + foreach ($model->getRules() as $rule) { + if(\is_array($rule['type'])) { + foreach ($rule['type'] as $type) { + $this->getUsedModels($type, $usedModels); + } + } else { + $this->getUsedModels($rule['type'], $usedModels); + } + } + } + /** * Parse * @@ -71,7 +98,7 @@ class OpenAPI3 extends Format if (isset($output['components']['securitySchemes']['Project'])) { $output['components']['securitySchemes']['Project']['x-appwrite'] = ['demo' => '5df5acd0d48c2']; } - + if (isset($output['components']['securitySchemes']['Key'])) { $output['components']['securitySchemes']['Key']['x-appwrite'] = ['demo' => '919c2d18fb5d4...a2ae413da83346ad2']; } @@ -79,7 +106,7 @@ class OpenAPI3 extends Format if (isset($output['securityDefinitions']['JWT'])) { $output['securityDefinitions']['JWT']['x-appwrite'] = ['demo' => 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJ...']; } - + if (isset($output['components']['securitySchemes']['Locale'])) { $output['components']['securitySchemes']['Locale']['x-appwrite'] = ['demo' => 'en']; } @@ -103,7 +130,7 @@ class OpenAPI3 extends Format $id = $route->getLabel('sdk.method', \uniqid()); $desc = (!empty($route->getLabel('sdk.description', ''))) ? \realpath(__DIR__.'/../../../../'.$route->getLabel('sdk.description', '')) : null; $produces = $route->getLabel('sdk.response.type', null); - $model = $route->getLabel('sdk.response.model', 'none'); + $model = $route->getLabel('sdk.response.model', 'none'); $routeSecurity = $route->getLabel('sdk.auth', []); $sdkPlatofrms = []; @@ -127,7 +154,7 @@ class OpenAPI3 extends Format if(empty($routeSecurity)) { $sdkPlatofrms[] = APP_PLATFORM_CLIENT; } - + $temp = [ 'summary' => $route->getDesc(), 'operationId' => $route->getLabel('sdk.namespace', 'default').ucfirst($id), @@ -153,13 +180,24 @@ class OpenAPI3 extends Format ]; foreach ($this->models as $key => $value) { - if($value->getType() === $model) { - $model = $value; - break; + if(\is_array($model)) { + $model = \array_map(function($m) use($value) { + if($m === $value->getType()) { + return $value; + } + + return $m; + }, $model); + } else { + if($value->getType() === $model) { + $model = $value; + break; + } } + } - if($model->isNone()) { + if(!(\is_array($model)) && $model->isNone()) { $temp['responses'][(string)$route->getLabel('sdk.response.code', '500')] = [ 'description' => (in_array($produces, [ 'image/*', @@ -176,17 +214,43 @@ class OpenAPI3 extends Format // ], ]; } else { - $usedModels[] = $model->getType(); - $temp['responses'][(string)$route->getLabel('sdk.response.code', '500')] = [ - 'description' => $model->getName(), - 'content' => [ - $produces => [ - 'schema' => [ - '$ref' => '#/components/schemas/'.$model->getType(), + if(\is_array($model)) { + $modelDescription = \join(', or ', \array_map(function ($m) { + return $m->getName(); + }, $model)); + + // model has multiple possible responses, we will use oneOf + foreach ($model as $m) { + $usedModels[] = $m->getType(); + } + + $temp['responses'][(string)$route->getLabel('sdk.response.code', '500')] = [ + 'description' => $modelDescription, + 'content' => [ + $produces => [ + 'schema' => [ + 'oneOf' => \array_map(function($m) { + return ['$ref' => '#/components/schemas/'.$m->getType()]; + }, $model) + ], ], ], - ], - ]; + ]; + } else { + // Response definition using one type + $usedModels[] = $model->getType(); + $temp['responses'][(string)$route->getLabel('sdk.response.code', '500')] = [ + 'description' => $model->getName(), + 'content' => [ + $produces => [ + 'schema' => [ + '$ref' => '#/components/schemas/'.$model->getType(), + ], + ], + ], + ]; + } + } if($route->getLabel('sdk.response.code', 500) === 204) { @@ -196,7 +260,7 @@ class OpenAPI3 extends Format if ((!empty($scope))) { // && 'public' != $scope $securities = ['Project' => []]; - + foreach($route->getLabel('sdk.auth', []) as $security) { if(array_key_exists($security, $this->keys)) { $securities[$security] = []; @@ -255,7 +319,7 @@ class OpenAPI3 extends Format case 'Utopia\Validator\JSON': case 'Utopia\Validator\Mock': case 'Utopia\Validator\Assoc': - $param['default'] = (empty($param['default'])) ? new stdClass() : $param['default']; + $param['default'] = (empty($param['default'])) ? new \stdClass() : $param['default']; $node['schema']['type'] = 'object'; $node['schema']['x-example'] = '{}'; //$node['schema']['format'] = 'json'; @@ -352,11 +416,7 @@ class OpenAPI3 extends Format $output['paths'][$url][\strtolower($route->getMethod())] = $temp; } foreach ($this->models as $model) { - foreach ($model->getRules() as $rule) { - if (!in_array($rule['type'], ['string', 'integer', 'boolean', 'json', 'float'])) { - $usedModels[] = $rule['type']; - } - } + $this->getUsedModels($model, $usedModels); } foreach ($this->models as $model) { if (!in_array($model->getType(), $usedModels) && $model->getType() !== 'error') { @@ -378,7 +438,7 @@ class OpenAPI3 extends Format if($model->isAny()) { $output['components']['schemas'][$model->getType()]['additionalProperties'] = true; } - + if(!empty($required)) { $output['components']['schemas'][$model->getType()]['required'] = $required; } @@ -393,7 +453,7 @@ class OpenAPI3 extends Format case 'json': $type = 'string'; break; - + case 'integer': $type = 'integer'; $format = 'int32'; @@ -403,18 +463,39 @@ class OpenAPI3 extends Format $type = 'number'; $format = 'float'; break; - + + case 'double': + $type = 'number'; + $format = 'double'; + break; + case 'boolean': $type = 'boolean'; break; - + default: $type = 'object'; $rule['type'] = ($rule['type']) ? $rule['type'] : 'none'; - $items = [ - '$ref' => '#/components/schemas/'.$rule['type'], - ]; + if(\is_array($rule['type'])) { + if($rule['array']) { + $items = [ + 'anyOf' => \array_map(function($type) { + return ['$ref' => '#/components/schemas/'.$type]; + }, $rule['type']) + ]; + } else { + $items = [ + 'oneOf' => \array_map(function($type) { + return ['$ref' => '#/components/schemas/'.$type]; + }, $rule['type']) + ]; + } + } else { + $items = [ + '$ref' => '#/components/schemas/'.$rule['type'], + ]; + } break; } diff --git a/src/Appwrite/Specification/Format/Swagger2.php b/src/Appwrite/Specification/Format/Swagger2.php index d357283a0..3ab5f184a 100644 --- a/src/Appwrite/Specification/Format/Swagger2.php +++ b/src/Appwrite/Specification/Format/Swagger2.php @@ -4,7 +4,6 @@ namespace Appwrite\Specification\Format; use Appwrite\Specification\Format; use Appwrite\Template\Template; -use stdClass; use Utopia\Validator; class Swagger2 extends Format @@ -21,6 +20,34 @@ class Swagger2 extends Format return 'Swagger 2'; } + /** + * Get Used Models + * + * Recursively get all used models + * + * @param object $model + * @param array $models + * + * @return void + */ + protected function getUsedModels($model, array &$usedModels) + { + if (is_string($model) && !in_array($model, ['string', 'integer', 'boolean', 'json', 'float', 'double'])) { + $usedModels[] = $model; + return; + } + if (!is_object($model)) return; + foreach ($model->getRules() as $rule) { + if(\is_array($rule['type'])) { + foreach ($rule['type'] as $type) { + $this->getUsedModels($type, $usedModels); + } + } else { + $this->getUsedModels($rule['type'], $usedModels); + } + } + } + /** * Parse * @@ -69,15 +96,15 @@ class Swagger2 extends Format if (isset($output['securityDefinitions']['Project'])) { $output['securityDefinitions']['Project']['x-appwrite'] = ['demo' => '5df5acd0d48c2']; } - + if (isset($output['securityDefinitions']['Key'])) { $output['securityDefinitions']['Key']['x-appwrite'] = ['demo' => '919c2d18fb5d4...a2ae413da83346ad2']; } - + if (isset($output['securityDefinitions']['JWT'])) { $output['securityDefinitions']['JWT']['x-appwrite'] = ['demo' => 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJ...']; } - + if (isset($output['securityDefinitions']['Locale'])) { $output['securityDefinitions']['Locale']['x-appwrite'] = ['demo' => 'en']; } @@ -125,7 +152,7 @@ class Swagger2 extends Format if(empty($routeSecurity)) { $sdkPlatofrms[] = APP_PLATFORM_CLIENT; } - + $temp = [ 'summary' => $route->getDesc(), 'operationId' => $route->getLabel('sdk.namespace', 'default').ucfirst($id), @@ -155,13 +182,22 @@ class Swagger2 extends Format } foreach ($this->models as $key => $value) { - if($value->getType() === $model) { - $model = $value; - break; + if(\is_array($model)) { + $model = \array_map(function($m) use($value) { + if($m === $value->getType()) { + return $value; + } + return $m; + }, $model); + } else { + if($value->getType() === $model) { + $model = $value; + break; + } } } - if($model->isNone()) { + if(!(\is_array($model)) && $model->isNone()) { $temp['responses'][(string)$route->getLabel('sdk.response.code', '500')] = [ 'description' => (in_array($produces, [ 'image/*', @@ -178,13 +214,41 @@ class Swagger2 extends Format ], ]; } else { - $usedModels[] = $model->getType(); - $temp['responses'][(string)$route->getLabel('sdk.response.code', '500')] = [ - 'description' => $model->getName(), - 'schema' => [ - '$ref' => '#/definitions/'.$model->getType(), - ], - ]; + + if(\is_array($model)) { + $modelDescription = \join(', or ', \array_map(function ($m) { + return $m->getName(); + }, $model)); + // model has multiple possible responses, we will use oneOf + foreach ($model as $m) { + $usedModels[] = $m->getType(); + } + $temp['responses'][(string)$route->getLabel('sdk.response.code', '500')] = [ + 'description' => $modelDescription, + 'content' => [ + $produces => [ + 'schema' => [ + 'oneOf' => \array_map(function($m) { + return ['$ref' => '#/definitions/'.$m->getType()]; + }, $model) + ], + ], + ], + ]; + } else { + // Response definition using one type + $usedModels[] = $model->getType(); + $temp['responses'][(string)$route->getLabel('sdk.response.code', '500')] = [ + 'description' => $model->getName(), + 'content' => [ + $produces => [ + 'schema' => [ + '$ref' => '#/definitions/'.$model->getType(), + ], + ], + ], + ]; + } } if(in_array($route->getLabel('sdk.response.code', 500), [204, 301, 302, 308], true)) { @@ -194,7 +258,7 @@ class Swagger2 extends Format if ((!empty($scope))) { // && 'public' != $scope $securities = ['Project' => []]; - + foreach($route->getLabel('sdk.auth', []) as $security) { if(array_key_exists($security, $this->keys)) { $securities[$security] = []; @@ -204,7 +268,7 @@ class Swagger2 extends Format $temp['x-appwrite']['auth'] = array_slice($securities, 0, $this->authCount); $temp['security'][] = $securities; } - + $body = [ 'name' => 'payload', 'in' => 'body', @@ -252,7 +316,7 @@ class Swagger2 extends Format case 'Utopia\Validator\Mock': case 'Utopia\Validator\Assoc': $node['type'] = 'object'; - $param['default'] = (empty($param['default'])) ? new stdClass() : $param['default']; + $param['default'] = (empty($param['default'])) ? new \stdClass() : $param['default']; $node['x-example'] = '{}'; //$node['format'] = 'json'; break; @@ -354,15 +418,9 @@ class Swagger2 extends Format $output['paths'][$url][\strtolower($route->getMethod())] = $temp; } foreach ($this->models as $model) { - foreach ($model->getRules() as $rule) { - if ( - in_array($model->getType(), $usedModels) - && !in_array($rule['type'], ['string', 'integer', 'boolean', 'json', 'float']) - ) { - $usedModels[] = $rule['type']; - } - } + $this->getUsedModels($model, $usedModels); } + foreach ($this->models as $model) { if (!in_array($model->getType(), $usedModels)) { continue; @@ -383,7 +441,7 @@ class Swagger2 extends Format if($model->isAny()) { $output['definitions'][$model->getType()]['additionalProperties'] = true; } - + if(!empty($required)) { $output['definitions'][$model->getType()]['required'] = $required; } @@ -398,7 +456,7 @@ class Swagger2 extends Format case 'json': $type = 'string'; break; - + case 'integer': $type = 'integer'; $format = 'int32'; @@ -408,19 +466,40 @@ class Swagger2 extends Format $type = 'number'; $format = 'float'; break; - + + case 'double': + $type = 'number'; + $format = 'double'; + break; + case 'boolean': $type = 'boolean'; break; - + default: $type = 'object'; $rule['type'] = ($rule['type']) ? $rule['type'] : 'none'; - $items = [ - 'type' => $type, - '$ref' => '#/definitions/'.$rule['type'], - ]; + if(\is_array($rule['type'])) { + if($rule['array']) { + $items = [ + 'anyOf' => \array_map(function($type) { + return ['$ref' => '#/definitions/'.$type]; + }, $rule['type']) + ]; + } else { + $items = [ + 'oneOf' => \array_map(function($type) { + return ['$ref' => '#/definitions/'.$type]; + }, $rule['type']) + ]; + } + } else { + $items = [ + 'type' => $type, + '$ref' => '#/definitions/'.$rule['type'], + ]; + } break; } diff --git a/src/Appwrite/Stats/Stats.php b/src/Appwrite/Stats/Stats.php index b07c4c85c..0cdd19fee 100644 --- a/src/Appwrite/Stats/Stats.php +++ b/src/Appwrite/Stats/Stats.php @@ -94,7 +94,7 @@ class Stats $functionExecutionTime = $this->params['functionExecutionTime'] ?? 0; $functionStatus = $this->params['functionStatus'] ?? ''; - $tags = ",project={$projectId},version=" . App::getEnv('_APP_VERSION', 'UNKNOWN'); + $tags = ",projectId={$projectId},version=" . App::getEnv('_APP_VERSION', 'UNKNOWN'); // the global namespace is prepended to every key (optional) $this->statsd->setNamespace($this->namespace); @@ -112,7 +112,70 @@ class Stats $this->statsd->count('network.outbound' . $tags, $networkResponseSize); $this->statsd->count('network.all' . $tags, $networkRequestSize + $networkResponseSize); + $dbMetrics = [ + 'database.collections.create', + 'database.collections.read', + 'database.collections.update', + 'database.collections.delete', + 'database.documents.create', + 'database.documents.read', + 'database.documents.update', + 'database.documents.delete', + ]; + + foreach ($dbMetrics as $metric) { + $value = $this->params[$metric] ?? 0; + if ($value >= 1) { + $tags = ",projectId={$projectId},collectionId=" . ($this->params['collectionId'] ?? ''); + $this->statsd->increment($metric . $tags); + } + } + + $storageMertics = [ + 'storage.files.create', + 'storage.files.read', + 'storage.files.update', + 'storage.files.delete', + ]; + + foreach ($storageMertics as $metric) { + $value = $this->params[$metric] ?? 0; + if ($value >= 1) { + $tags = ",projectId={$projectId},bucketId=" . ($this->params['bucketId'] ?? ''); + $this->statsd->increment($metric . $tags); + } + } + + $usersMetrics = [ + 'users.create', + 'users.read', + 'users.update', + 'users.delete', + ]; + + foreach ($usersMetrics as $metric) { + $value = $this->params[$metric] ?? 0; + if ($value >= 1) { + $tags = ",projectId={$projectId}"; + $this->statsd->increment($metric . $tags); + } + } + + $sessionsMetrics = [ + 'users.sessions.create', + 'users.sessions.delete', + ]; + + foreach ($sessionsMetrics as $metric) { + $value = $this->params[$metric] ?? 0; + if ($value >= 1) { + $tags = ",projectId={$projectId},provider=". ($this->params['provider'] ?? ''); + $this->statsd->count($metric . $tags, $value); + } + } + if ($storage >= 1) { + $tags = ",projectId={$projectId},bucketId=" . ($this->params['bucketId'] ?? ''); $this->statsd->count('storage.all' . $tags, $storage); } diff --git a/src/Appwrite/Utopia/Response.php b/src/Appwrite/Utopia/Response.php index 6a1ff2307..61511138f 100644 --- a/src/Appwrite/Utopia/Response.php +++ b/src/Appwrite/Utopia/Response.php @@ -42,6 +42,7 @@ use Appwrite\Utopia\Response\Model\Team; use Appwrite\Utopia\Response\Model\Locale; use Appwrite\Utopia\Response\Model\Log; use Appwrite\Utopia\Response\Model\Membership; +use Appwrite\Utopia\Response\Model\Metric; use Appwrite\Utopia\Response\Model\Permissions; use Appwrite\Utopia\Response\Model\Phone; use Appwrite\Utopia\Response\Model\Platform; @@ -52,7 +53,13 @@ use Appwrite\Utopia\Response\Model\Token; use Appwrite\Utopia\Response\Model\Webhook; use Appwrite\Utopia\Response\Model\Preferences; use Appwrite\Utopia\Response\Model\Mock; // Keep last -use stdClass; +use Appwrite\Utopia\Response\Model\UsageBuckets; +use Appwrite\Utopia\Response\Model\UsageCollection; +use Appwrite\Utopia\Response\Model\UsageDatabase; +use Appwrite\Utopia\Response\Model\UsageFunctions; +use Appwrite\Utopia\Response\Model\UsageProject; +use Appwrite\Utopia\Response\Model\UsageStorage; +use Appwrite\Utopia\Response\Model\UsageUsers; /** * @method Response public function setStatusCode(int $code = 200) @@ -65,8 +72,17 @@ class Response extends SwooleResponse const MODEL_LOG = 'log'; const MODEL_LOG_LIST = 'logList'; const MODEL_ERROR = 'error'; + const MODEL_METRIC = 'metric'; + const MODEL_METRIC_LIST = 'metricList'; const MODEL_ERROR_DEV = 'errorDev'; const MODEL_BASE_LIST = 'baseList'; + const MODEL_USAGE_DATABASE = 'usageDatabase'; + const MODEL_USAGE_COLLECTION = 'usageCollection'; + const MODEL_USAGE_USERS = 'usageUsers'; + const MODEL_USAGE_BUCKETS = 'usageBuckets'; + const MODEL_USAGE_STORAGE = 'usageStorage'; + const MODEL_USAGE_FUNCTIONS = 'usageFunctions'; + const MODEL_USAGE_PROJECT = 'usageProject'; // Database const MODEL_COLLECTION = 'collection'; @@ -144,6 +160,7 @@ class Response extends SwooleResponse // Deprecated const MODEL_PERMISSIONS = 'permissions'; const MODEL_RULE = 'rule'; + const MODEL_TASK = 'task'; // Tests (keep last) const MODEL_MOCK = 'mock'; @@ -194,6 +211,7 @@ class Response extends SwooleResponse ->setModel(new BaseList('Languages List', self::MODEL_LANGUAGE_LIST, 'languages', self::MODEL_LANGUAGE)) ->setModel(new BaseList('Currencies List', self::MODEL_CURRENCY_LIST, 'currencies', self::MODEL_CURRENCY)) ->setModel(new BaseList('Phones List', self::MODEL_PHONE_LIST, 'phones', self::MODEL_PHONE)) + ->setModel(new BaseList('Metric List', self::MODEL_METRIC_LIST, 'metrics', self::MODEL_METRIC, true, false)) // Entities ->setModel(new Collection()) ->setModel(new Attribute()) @@ -231,6 +249,14 @@ class Response extends SwooleResponse ->setModel(new Language()) ->setModel(new Currency()) ->setModel(new Phone()) + ->setModel(new Metric()) + ->setModel(new UsageDatabase()) + ->setModel(new UsageCollection()) + ->setModel(new UsageUsers()) + ->setModel(new UsageStorage()) + ->setModel(new UsageBuckets()) + ->setModel(new UsageFunctions()) + ->setModel(new UsageProject()) // Verification // Recovery // Tests (keep last) @@ -304,7 +330,7 @@ class Response extends SwooleResponse $output = self::getFilter()->parse($output, $model); } - $this->json(!empty($output) ? $output : new stdClass()); + $this->json(!empty($output) ? $output : new \stdClass()); } /** @@ -344,7 +370,24 @@ class Response extends SwooleResponse foreach ($data[$key] as &$item) { if ($item instanceof Document) { - $ruleType = (!\is_null($rule['getNestedType'])) ? $rule['getNestedType']($item) : $rule['type']; + if (\is_array($rule['type'])) { + foreach ($rule['type'] as $type) { + $condition = false; + foreach ($this->getModel($type)->conditions as $attribute => $val) { + $condition = $item->getAttribute($attribute) === $val; + if(!$condition) { + break; + } + } + if ($condition) { + $ruleType = $type; + break; + } + } + } else { + $ruleType = $rule['type']; + } + if (!array_key_exists($ruleType, $this->models)) { throw new Exception('Missing model for rule: '. $ruleType); } @@ -353,7 +396,7 @@ class Response extends SwooleResponse } } } - + $output[$key] = $data[$key]; } diff --git a/src/Appwrite/Utopia/Response/Model.php b/src/Appwrite/Utopia/Response/Model.php index a763692c7..4c3143ca9 100644 --- a/src/Appwrite/Utopia/Response/Model.php +++ b/src/Appwrite/Utopia/Response/Model.php @@ -69,13 +69,11 @@ abstract class Model /** * Add a New Rule * If rule is an array of documents with varying models - * Pass callable $getNestedType that accepts Document and returns the nested response type * * @param string $key * @param array $options - * @param callable $getNestedType function(Document $value): string */ - protected function addRule(string $key, array $options, callable $getNestedType = null): self + protected function addRule(string $key, array $options): self { $this->rules[$key] = array_merge([ 'require' => true, @@ -83,8 +81,7 @@ abstract class Model 'description' => '', 'default' => null, 'example' => '', - 'array' => false, - 'getNestedType' => $getNestedType + 'array' => false ], $options); return $this; diff --git a/src/Appwrite/Utopia/Response/Model/Attribute.php b/src/Appwrite/Utopia/Response/Model/Attribute.php index 2c9dbf7c2..281a37a73 100644 --- a/src/Appwrite/Utopia/Response/Model/Attribute.php +++ b/src/Appwrite/Utopia/Response/Model/Attribute.php @@ -44,6 +44,8 @@ class Attribute extends Model ; } + public array $conditions = []; + /** * Get Name * diff --git a/src/Appwrite/Utopia/Response/Model/AttributeBoolean.php b/src/Appwrite/Utopia/Response/Model/AttributeBoolean.php index 4076bc9eb..de7ae5813 100644 --- a/src/Appwrite/Utopia/Response/Model/AttributeBoolean.php +++ b/src/Appwrite/Utopia/Response/Model/AttributeBoolean.php @@ -23,6 +23,10 @@ class AttributeBoolean extends Attribute ; } + public array $conditions = [ + 'type' => self::TYPE_BOOLEAN + ]; + /** * Get Name * @@ -34,7 +38,7 @@ class AttributeBoolean extends Attribute } /** - * Get Collection + * Get Type * * @return string */ diff --git a/src/Appwrite/Utopia/Response/Model/AttributeEmail.php b/src/Appwrite/Utopia/Response/Model/AttributeEmail.php index 15bc4674e..268412a55 100644 --- a/src/Appwrite/Utopia/Response/Model/AttributeEmail.php +++ b/src/Appwrite/Utopia/Response/Model/AttributeEmail.php @@ -31,6 +31,11 @@ class AttributeEmail extends Attribute ; } + public array $conditions = [ + 'type' => self::TYPE_STRING, + 'format' => \APP_DATABASE_ATTRIBUTE_EMAIL + ]; + /** * Get Name * @@ -42,7 +47,7 @@ class AttributeEmail extends Attribute } /** - * Get Collection + * Get Type * * @return string */ diff --git a/src/Appwrite/Utopia/Response/Model/AttributeFloat.php b/src/Appwrite/Utopia/Response/Model/AttributeFloat.php index 7c4ead9e7..93033f5b0 100644 --- a/src/Appwrite/Utopia/Response/Model/AttributeFloat.php +++ b/src/Appwrite/Utopia/Response/Model/AttributeFloat.php @@ -39,6 +39,10 @@ class AttributeFloat extends Attribute ; } + public array $conditions = [ + 'type' => self::TYPE_FLOAT, + ]; + /** * Get Name * @@ -50,7 +54,7 @@ class AttributeFloat extends Attribute } /** - * Get Collection + * Get Type * * @return string */ diff --git a/src/Appwrite/Utopia/Response/Model/AttributeIP.php b/src/Appwrite/Utopia/Response/Model/AttributeIP.php index ec3a9cfed..2b8f64ded 100644 --- a/src/Appwrite/Utopia/Response/Model/AttributeIP.php +++ b/src/Appwrite/Utopia/Response/Model/AttributeIP.php @@ -31,6 +31,11 @@ class AttributeIP extends Attribute ; } + public array $conditions = [ + 'type' => self::TYPE_STRING, + 'format' => \APP_DATABASE_ATTRIBUTE_IP + ]; + /** * Get Name * @@ -42,7 +47,7 @@ class AttributeIP extends Attribute } /** - * Get Collection + * Get Type * * @return string */ diff --git a/src/Appwrite/Utopia/Response/Model/AttributeInteger.php b/src/Appwrite/Utopia/Response/Model/AttributeInteger.php index f11344d79..9ff94c8f5 100644 --- a/src/Appwrite/Utopia/Response/Model/AttributeInteger.php +++ b/src/Appwrite/Utopia/Response/Model/AttributeInteger.php @@ -39,6 +39,10 @@ class AttributeInteger extends Attribute ; } + public array $conditions = [ + 'type' => self::TYPE_INTEGER, + ]; + /** * Get Name * * @return string @@ -49,7 +53,7 @@ class AttributeInteger extends Attribute } /** - * Get Collection + * Get Type * * @return string */ diff --git a/src/Appwrite/Utopia/Response/Model/AttributeList.php b/src/Appwrite/Utopia/Response/Model/AttributeList.php index 69e083ced..0cf67b3bd 100644 --- a/src/Appwrite/Utopia/Response/Model/AttributeList.php +++ b/src/Appwrite/Utopia/Response/Model/AttributeList.php @@ -18,25 +18,18 @@ class AttributeList extends Model 'example' => 5, ]) ->addRule('attributes', [ - 'type' => Response::MODEL_ATTRIBUTE, + 'type' => [ + Response::MODEL_ATTRIBUTE_BOOLEAN, + Response::MODEL_ATTRIBUTE_INTEGER, + Response::MODEL_ATTRIBUTE_FLOAT, + Response::MODEL_ATTRIBUTE_EMAIL, + Response::MODEL_ATTRIBUTE_URL, + Response::MODEL_ATTRIBUTE_IP, + Response::MODEL_ATTRIBUTE_STRING // needs to be last, since its condition would dominate any other string attribute + ], 'description' => 'List of attributes.', 'default' => [], - 'array' => true, - 'getNestedType' => function(Document $attribute) { - return match($attribute->getAttribute('type')) { - self::TYPE_BOOLEAN => Response::MODEL_ATTRIBUTE_BOOLEAN, - self::TYPE_INTEGER => Response::MODEL_ATTRIBUTE_INTEGER, - self::TYPE_FLOAT => Response::MODEL_ATTRIBUTE_FLOAT, - self::TYPE_STRING => match($attribute->getAttribute('format')) { - APP_DATABASE_ATTRIBUTE_EMAIL => Response::MODEL_ATTRIBUTE_EMAIL, - APP_DATABASE_ATTRIBUTE_ENUM => Response::MODEL_ATTRIBUTE_ENUM, - APP_DATABASE_ATTRIBUTE_IP => Response::MODEL_ATTRIBUTE_IP, - APP_DATABASE_ATTRIBUTE_URL => Response::MODEL_ATTRIBUTE_URL, - default => Response::MODEL_ATTRIBUTE_STRING, - }, - default => Response::MODEL_ATTRIBUTE, - }; - }, + 'array' => true ]) ; } @@ -52,7 +45,7 @@ class AttributeList extends Model } /** - * Get Collection + * Get Type * * @return string */ diff --git a/src/Appwrite/Utopia/Response/Model/AttributeString.php b/src/Appwrite/Utopia/Response/Model/AttributeString.php index 22f7e52a3..e7ea85326 100644 --- a/src/Appwrite/Utopia/Response/Model/AttributeString.php +++ b/src/Appwrite/Utopia/Response/Model/AttributeString.php @@ -29,6 +29,10 @@ class AttributeString extends Attribute ; } + public array $conditions = [ + 'type' => self::TYPE_STRING, + ]; + /** * Get Name * @@ -40,7 +44,7 @@ class AttributeString extends Attribute } /** - * Get Collection + * Get Type * * @return string */ diff --git a/src/Appwrite/Utopia/Response/Model/AttributeURL.php b/src/Appwrite/Utopia/Response/Model/AttributeURL.php index 3caddbc10..489e03885 100644 --- a/src/Appwrite/Utopia/Response/Model/AttributeURL.php +++ b/src/Appwrite/Utopia/Response/Model/AttributeURL.php @@ -31,6 +31,11 @@ class AttributeURL extends Attribute ; } + public array $conditions = [ + 'type' => self::TYPE_STRING, + 'format' => \APP_DATABASE_ATTRIBUTE_URL + ]; + /** * Get Name * @@ -42,7 +47,7 @@ class AttributeURL extends Attribute } /** - * Get Collection + * Get Type * * @return string */ diff --git a/src/Appwrite/Utopia/Response/Model/Collection.php b/src/Appwrite/Utopia/Response/Model/Collection.php index f59674bbf..b2a05846c 100644 --- a/src/Appwrite/Utopia/Response/Model/Collection.php +++ b/src/Appwrite/Utopia/Response/Model/Collection.php @@ -4,8 +4,6 @@ namespace Appwrite\Utopia\Response\Model; use Appwrite\Utopia\Response; use Appwrite\Utopia\Response\Model; -use Utopia\Database\Document; -use stdClass; class Collection extends Model { @@ -45,32 +43,25 @@ class Collection extends Model 'example' => 'document', ]) ->addRule('attributes', [ - 'type' => Response::MODEL_ATTRIBUTE, + 'type' => [ + Response::MODEL_ATTRIBUTE_BOOLEAN, + Response::MODEL_ATTRIBUTE_INTEGER, + Response::MODEL_ATTRIBUTE_FLOAT, + Response::MODEL_ATTRIBUTE_EMAIL, + Response::MODEL_ATTRIBUTE_URL, + Response::MODEL_ATTRIBUTE_IP, + Response::MODEL_ATTRIBUTE_STRING, // needs to be last, since its condition would dominate any other string attribute + ], 'description' => 'Collection attributes.', 'default' => [], - 'example' => new stdClass, + 'example' => new \stdClass, 'array' => true, - 'getNestedType' => function(Document $attribute) { - return match($attribute->getAttribute('type')) { - self::TYPE_BOOLEAN => Response::MODEL_ATTRIBUTE_BOOLEAN, - self::TYPE_INTEGER => Response::MODEL_ATTRIBUTE_INTEGER, - self::TYPE_FLOAT => Response::MODEL_ATTRIBUTE_FLOAT, - self::TYPE_STRING => match($attribute->getAttribute('format')) { - APP_DATABASE_ATTRIBUTE_EMAIL => Response::MODEL_ATTRIBUTE_EMAIL, - APP_DATABASE_ATTRIBUTE_ENUM => Response::MODEL_ATTRIBUTE_ENUM, - APP_DATABASE_ATTRIBUTE_IP => Response::MODEL_ATTRIBUTE_IP, - APP_DATABASE_ATTRIBUTE_URL => Response::MODEL_ATTRIBUTE_URL, - default => Response::MODEL_ATTRIBUTE_STRING, - }, - default => Response::MODEL_ATTRIBUTE, - }; - }, ]) ->addRule('indexes', [ 'type' => Response::MODEL_INDEX, 'description' => 'Collection indexes.', 'default' => [], - 'example' => new stdClass, + 'example' => new \stdClass, 'array' => true ]) ; diff --git a/src/Appwrite/Utopia/Response/Model/Metric.php b/src/Appwrite/Utopia/Response/Model/Metric.php new file mode 100644 index 000000000..26038af08 --- /dev/null +++ b/src/Appwrite/Utopia/Response/Model/Metric.php @@ -0,0 +1,47 @@ +addRule('value', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'The value of this metric at the timestamp.', + 'default' => -1, + 'example' => 1, + ]) + ->addRule('timestamp', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'The UNIX timestamp at which this metric was aggregated.', + 'default' => 0, + 'example' => 1592981250 + ]) + ; + } + + /** + * Get Name + * + * @return string + */ + public function getName():string + { + return 'Metric'; + } + + /** + * Get Collection + * + * @return string + */ + public function getType():string + { + return Response::MODEL_METRIC; + } +} \ No newline at end of file diff --git a/src/Appwrite/Utopia/Response/Model/Project.php b/src/Appwrite/Utopia/Response/Model/Project.php index adbd92266..ed2fd60f1 100644 --- a/src/Appwrite/Utopia/Response/Model/Project.php +++ b/src/Appwrite/Utopia/Response/Model/Project.php @@ -2,7 +2,6 @@ namespace Appwrite\Utopia\Response\Model; -use stdClass; use Appwrite\Utopia\Response; use Appwrite\Utopia\Response\Model; use Utopia\Config\Config; @@ -100,28 +99,28 @@ class Project extends Model 'type' => Response::MODEL_PLATFORM, 'description' => 'List of Platforms.', 'default' => [], - 'example' => new stdClass, + 'example' => new \stdClass, 'array' => true, ]) ->addRule('webhooks', [ 'type' => Response::MODEL_WEBHOOK, 'description' => 'List of Webhooks.', 'default' => [], - 'example' => new stdClass, + 'example' => new \stdClass, 'array' => true, ]) ->addRule('keys', [ 'type' => Response::MODEL_KEY, 'description' => 'List of API Keys.', 'default' => [], - 'example' => new stdClass, + 'example' => new \stdClass, 'array' => true, ]) ->addRule('domains', [ 'type' => Response::MODEL_DOMAIN, 'description' => 'List of Domains.', 'default' => [], - 'example' => new stdClass, + 'example' => new \stdClass, 'array' => true, ]) ; diff --git a/src/Appwrite/Utopia/Response/Model/UsageBuckets.php b/src/Appwrite/Utopia/Response/Model/UsageBuckets.php new file mode 100644 index 000000000..cafa679ab --- /dev/null +++ b/src/Appwrite/Utopia/Response/Model/UsageBuckets.php @@ -0,0 +1,76 @@ +addRule('range', [ + 'type' => self::TYPE_STRING, + 'description' => 'The time range of the usage stats.', + 'default' => '', + 'example' => '30d', + ]) + ->addRule('files.count', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for total number of files in this bucket.', + 'default' => [], + 'example' => new \stdClass, + 'array' => true + ]) + ->addRule('files.create', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for files created.', + 'default' => [], + 'example' => new \stdClass, + 'array' => true + ]) + ->addRule('files.read', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for files read.', + 'default' => [], + 'example' => new \stdClass, + 'array' => true + ]) + ->addRule('files.update', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for files updated.', + 'default' => [], + 'example' => new \stdClass, + 'array' => true + ]) + ->addRule('files.delete', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for files deleted.', + 'default' => [], + 'example' => new \stdClass, + 'array' => true + ]) + ; + } + + /** + * Get Name + * + * @return string + */ + public function getName():string + { + return 'UsageBuckets'; + } + + /** + * Get Type + * + * @return string + */ + public function getType():string + { + return Response::MODEL_USAGE_BUCKETS; + } +} \ No newline at end of file diff --git a/src/Appwrite/Utopia/Response/Model/UsageCollection.php b/src/Appwrite/Utopia/Response/Model/UsageCollection.php new file mode 100644 index 000000000..cf9e3e454 --- /dev/null +++ b/src/Appwrite/Utopia/Response/Model/UsageCollection.php @@ -0,0 +1,76 @@ +addRule('range', [ + 'type' => self::TYPE_STRING, + 'description' => 'The time range of the usage stats.', + 'default' => '', + 'example' => '30d', + ]) + ->addRule('documents.count', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for total number of documents.', + 'default' => [], + 'example' => new \stdClass, + 'array' => true + ]) + ->addRule('documents.create', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for documents created.', + 'default' => [], + 'example' => new \stdClass, + 'array' => true + ]) + ->addRule('documents.read', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for documents read.', + 'default' => [], + 'example' => new \stdClass, + 'array' => true + ]) + ->addRule('documents.update', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for documents updated.', + 'default' => [], + 'example' => new \stdClass, + 'array' => true + ]) + ->addRule('documents.delete', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for documents deleted.', + 'default' => [], + 'example' => new \stdClass, + 'array' => true + ]) + ; + } + + /** + * Get Name + * + * @return string + */ + public function getName():string + { + return 'UsageCollection'; + } + + /** + * Get Type + * + * @return string + */ + public function getType():string + { + return Response::MODEL_USAGE_COLLECTION; + } +} \ No newline at end of file diff --git a/src/Appwrite/Utopia/Response/Model/UsageDatabase.php b/src/Appwrite/Utopia/Response/Model/UsageDatabase.php new file mode 100644 index 000000000..eab025ba1 --- /dev/null +++ b/src/Appwrite/Utopia/Response/Model/UsageDatabase.php @@ -0,0 +1,111 @@ +addRule('range', [ + 'type' => self::TYPE_STRING, + 'description' => 'The time range of the usage stats.', + 'default' => '', + 'example' => '30d', + ]) + ->addRule('documents.count', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for total number of documents.', + 'default' => [], + 'example' => new \stdClass, + 'array' => true + ]) + ->addRule('collections.count', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for total number of collections.', + 'default' => [], + 'example' => new \stdClass, + 'array' => true + ]) + ->addRule('documents.create', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for documents created.', + 'default' => [], + 'example' => new \stdClass, + 'array' => true + ]) + ->addRule('documents.read', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for documents read.', + 'default' => [], + 'example' => new \stdClass, + 'array' => true + ]) + ->addRule('documents.update', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for documents updated.', + 'default' => [], + 'example' => new \stdClass, + 'array' => true + ]) + ->addRule('documents.delete', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for documents deleted.', + 'default' => [], + 'example' => new \stdClass, + 'array' => true + ]) + ->addRule('collections.create', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for collections created.', + 'default' => [], + 'example' => new \stdClass, + 'array' => true + ]) + ->addRule('collections.read', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for collections read.', + 'default' => [], + 'example' => new \stdClass, + 'array' => true + ]) + ->addRule('collections.update', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for collections updated.', + 'default' => [], + 'example' => new \stdClass, + 'array' => true + ]) + ->addRule('collections.delete', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for collections delete.', + 'default' => [], + 'example' => new \stdClass, + 'array' => true + ]) + ; + } + + /** + * Get Name + * + * @return string + */ + public function getName():string + { + return 'UsageDatabase'; + } + + /** + * Get Type + * + * @return string + */ + public function getType():string + { + return Response::MODEL_USAGE_DATABASE; + } +} \ No newline at end of file diff --git a/src/Appwrite/Utopia/Response/Model/UsageFunctions.php b/src/Appwrite/Utopia/Response/Model/UsageFunctions.php new file mode 100644 index 000000000..6183b1aff --- /dev/null +++ b/src/Appwrite/Utopia/Response/Model/UsageFunctions.php @@ -0,0 +1,62 @@ +addRule('range', [ + 'type' => self::TYPE_STRING, + 'description' => 'The time range of the usage stats.', + 'default' => '', + 'example' => '30d', + ]) + ->addRule('functions.executions', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for function executions.', + 'default' => [], + 'example' => new \stdClass, + 'array' => true + ]) + ->addRule('functions.failures', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for function execution failures.', + 'default' => [], + 'example' => new \stdClass, + 'array' => true + ]) + ->addRule('functions.compute', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for function execution duration.', + 'default' => [], + 'example' => new \stdClass, + 'array' => true + ]) + ; + } + + /** + * Get Name + * + * @return string + */ + public function getName():string + { + return 'UsageFunctions'; + } + + /** + * Get Type + * + * @return string + */ + public function getType():string + { + return Response::MODEL_USAGE_FUNCTIONS; + } +} \ No newline at end of file diff --git a/src/Appwrite/Utopia/Response/Model/UsageProject.php b/src/Appwrite/Utopia/Response/Model/UsageProject.php new file mode 100644 index 000000000..5593def1d --- /dev/null +++ b/src/Appwrite/Utopia/Response/Model/UsageProject.php @@ -0,0 +1,90 @@ +addRule('range', [ + 'type' => self::TYPE_STRING, + 'description' => 'The time range of the usage stats.', + 'default' => '', + 'example' => '30d', + ]) + ->addRule('requests', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for number of requests.', + 'default' => [], + 'example' => new \stdClass, + 'array' => true + ]) + ->addRule('network', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for consumed bandwidth.', + 'default' => [], + 'example' => new \stdClass, + 'array' => true + ]) + ->addRule('functions', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for function executions.', + 'default' => [], + 'example' => new \stdClass, + 'array' => true + ]) + ->addRule('documents', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for number of documents.', + 'default' => [], + 'example' => new \stdClass, + 'array' => true + ]) + ->addRule('collections', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for number of collections.', + 'default' => [], + 'example' => new \stdClass, + 'array' => true + ]) + ->addRule('users', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for number of users.', + 'default' => [], + 'example' => new \stdClass, + 'array' => true + ]) + ->addRule('storage', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for the occupied storage size (in bytes).', + 'default' => [], + 'example' => new \stdClass, + 'array' => true + ]) + ; + } + + /** + * Get Name + * + * @return string + */ + public function getName():string + { + return 'UsageProject'; + } + + /** + * Get Type + * + * @return string + */ + public function getType():string + { + return Response::MODEL_USAGE_PROJECT; + } +} \ No newline at end of file diff --git a/src/Appwrite/Utopia/Response/Model/UsageStorage.php b/src/Appwrite/Utopia/Response/Model/UsageStorage.php new file mode 100644 index 000000000..db40ccf3b --- /dev/null +++ b/src/Appwrite/Utopia/Response/Model/UsageStorage.php @@ -0,0 +1,55 @@ +addRule('range', [ + 'type' => self::TYPE_STRING, + 'description' => 'The time range of the usage stats.', + 'default' => '', + 'example' => '30d', + ]) + ->addRule('storage', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for the occupied storage size (in bytes).', + 'default' => [], + 'example' => new \stdClass, + 'array' => true + ]) + ->addRule('files', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for total number of files.', + 'default' => [], + 'example' => new \stdClass, + 'array' => true + ]) + ; + } + + /** + * Get Name + * + * @return string + */ + public function getName():string + { + return 'StorageUsage'; + } + + /** + * Get Type + * + * @return string + */ + public function getType():string + { + return Response::MODEL_USAGE_STORAGE; + } +} \ No newline at end of file diff --git a/src/Appwrite/Utopia/Response/Model/UsageUsers.php b/src/Appwrite/Utopia/Response/Model/UsageUsers.php new file mode 100644 index 000000000..3854fdec7 --- /dev/null +++ b/src/Appwrite/Utopia/Response/Model/UsageUsers.php @@ -0,0 +1,97 @@ +addRule('range', [ + 'type' => self::TYPE_STRING, + 'description' => 'The time range of the usage stats.', + 'default' => '', + 'example' => '30d', + ]) + ->addRule('users.count', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for total number of users.', + 'default' => [], + 'example' => new \stdClass, + 'array' => true + ]) + ->addRule('users.create', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for users created.', + 'default' => [], + 'example' => new \stdClass, + 'array' => true + ]) + ->addRule('users.read', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for users read.', + 'default' => [], + 'example' => new \stdClass, + 'array' => true + ]) + ->addRule('users.update', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for users updated.', + 'default' => [], + 'example' => new \stdClass, + 'array' => true + ]) + ->addRule('users.delete', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for users deleted.', + 'default' => [], + 'example' => new \stdClass, + 'array' => true + ]) + ->addRule('sessions.create', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for sessions created.', + 'default' => [], + 'example' => new \stdClass, + 'array' => true + ]) + ->addRule('sessions.provider.create', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for sessions created for a provider ( email, anonymous or oauth2 ).', + 'default' => [], + 'example' => new \stdClass, + 'array' => true + ]) + ->addRule('sessions.delete', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for sessions deleted.', + 'default' => [], + 'example' => new \stdClass, + 'array' => true + ]) + ; + } + + /** + * Get Name + * + * @return string + */ + public function getName():string + { + return 'UsageUsers'; + } + + /** + * Get Type + * + * @return string + */ + public function getType():string + { + return Response::MODEL_USAGE_USERS; + } +} \ No newline at end of file diff --git a/tests/e2e/Scopes/SideConsole.php b/tests/e2e/Scopes/SideConsole.php new file mode 100644 index 000000000..ada99a6e6 --- /dev/null +++ b/tests/e2e/Scopes/SideConsole.php @@ -0,0 +1,23 @@ + 'http://localhost', + 'cookie' => 'a_session_console='. $this->getRoot()['session'], + 'x-appwrite-mode' => 'admin' + ]; + } + + /** + * @return string + */ + public function getSide() + { + return 'console'; + } +} diff --git a/tests/e2e/Services/Database/DatabaseBase.php b/tests/e2e/Services/Database/DatabaseBase.php index f77538ed6..e62f7fb7a 100644 --- a/tests/e2e/Services/Database/DatabaseBase.php +++ b/tests/e2e/Services/Database/DatabaseBase.php @@ -1234,7 +1234,7 @@ trait DatabaseBase // $this->assertEquals('Minimum value must be lesser than maximum value', $invalidRange['body']['message']); // wait for worker to add attributes - sleep(2); + sleep(3); $collection = $this->client->call(Client::METHOD_GET, '/database/collections/' . $collectionId, array_merge([ 'content-type' => 'application/json', @@ -1597,6 +1597,144 @@ trait DatabaseBase return $data; } + + public function testEnforceCollectionPermissions() + { + $user = 'user:' . $this->getUser()['$id']; + $collection = $this->client->call(Client::METHOD_POST, '/database/collections', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'collectionId' => 'unique()', + 'name' => 'enforceCollectionPermissions', + 'permission' => 'collection', + 'read' => [$user], + 'write' => [$user] + ]); + + $this->assertEquals($collection['headers']['status-code'], 201); + $this->assertEquals($collection['body']['name'], 'enforceCollectionPermissions'); + $this->assertEquals($collection['body']['permission'], 'collection'); + + $collectionId = $collection['body']['$id']; + + sleep(2); + + $attribute = $this->client->call(Client::METHOD_POST, '/database/collections/' . $collectionId . '/attributes/string', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'attributeId' => 'attribute', + 'size' => 64, + 'required' => true, + ]); + + $this->assertEquals(201, $attribute['headers']['status-code'], 201); + $this->assertEquals('attribute', $attribute['body']['key']); + + // wait for db to add attribute + sleep(2); + + $index = $this->client->call(Client::METHOD_POST, '/database/collections/' . $collectionId . '/indexes', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'indexId' => 'key_attribute', + 'type' => 'key', + 'attributes' => [$attribute['body']['key']], + ]); + + $this->assertEquals(201, $index['headers']['status-code']); + $this->assertEquals('key_attribute', $index['body']['key']); + + // wait for db to add attribute + sleep(2); + + $document1 = $this->client->call(Client::METHOD_POST, '/database/collections/' . $collectionId . '/documents', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'documentId' => 'unique()', + 'data' => [ + 'attribute' => 'one', + ], + 'read' => [$user], + 'write' => [$user], + ]); + + $this->assertEquals(201, $document1['headers']['status-code']); + + $documents = $this->client->call(Client::METHOD_GET, '/database/collections/' . $collectionId . '/documents', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(1, $documents['body']['sum']); + $this->assertCount(1, $documents['body']['documents']); + + /* + * Test for Failure + */ + + // Remove write permission + $collection = $this->client->call(Client::METHOD_PUT, '/database/collections/' . $collectionId, array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'name' => 'enforceCollectionPermissions', + 'permission' => 'collection', + 'read' => [$user], + 'write' => [] + ]); + + $this->assertEquals(200, $collection['headers']['status-code']); + + $badDocument = $this->client->call(Client::METHOD_POST, '/database/collections/' . $collectionId . '/documents', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'documentId' => 'unique()', + 'data' => [ + 'attribute' => 'bad', + ], + 'read' => [$user], + 'write' => [$user], + ]); + + if($this->getSide() == 'client') { + $this->assertEquals(401, $badDocument['headers']['status-code']); + } + + if($this->getSide() == 'server') { + $this->assertEquals(201, $badDocument['headers']['status-code']); + } + + // Remove read permission + $collection = $this->client->call(Client::METHOD_PUT, '/database/collections/' . $collectionId, array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'name' => 'enforceCollectionPermissions', + 'permission' => 'collection', + 'read' => [], + 'write' => [] + ]); + + $this->assertEquals(200, $collection['headers']['status-code']); + + $documents = $this->client->call(Client::METHOD_GET, '/database/collections/' . $collectionId . '/documents', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ])); + + $this->assertEquals(404, $documents['headers']['status-code']); + } + /** * @depends testDefaultPermissions */ diff --git a/tests/e2e/Services/Database/DatabaseConsoleClientTest.php b/tests/e2e/Services/Database/DatabaseConsoleClientTest.php new file mode 100644 index 000000000..625f26107 --- /dev/null +++ b/tests/e2e/Services/Database/DatabaseConsoleClientTest.php @@ -0,0 +1,125 @@ +client->call(Client::METHOD_POST, '/database/collections', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'collectionId' => 'unique()', + 'name' => 'Movies', + 'read' => ['role:all'], + 'write' => ['role:all'], + 'permission' => 'document', + ]); + + $this->assertEquals($movies['headers']['status-code'], 201); + $this->assertEquals($movies['body']['name'], 'Movies'); + + return ['moviesId' => $movies['body']['$id']]; + } + + public function testGetDatabaseUsage() + { + /** + * Test for FAILURE + */ + + $response = $this->client->call(Client::METHOD_GET, '/database/usage', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), [ + 'range' => '32h' + ]); + + $this->assertEquals($response['headers']['status-code'], 400); + + /** + * Test for SUCCESS + */ + + $response = $this->client->call(Client::METHOD_GET, '/database/usage', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), [ + 'range' => '24h' + ]); + + $this->assertEquals($response['headers']['status-code'], 200); + $this->assertEquals(count($response['body']), 11); + $this->assertEquals($response['body']['range'], '24h'); + $this->assertIsArray($response['body']['documents.count']); + $this->assertIsArray($response['body']['collections.count']); + $this->assertIsArray($response['body']['documents.create']); + $this->assertIsArray($response['body']['documents.read']); + $this->assertIsArray($response['body']['documents.update']); + $this->assertIsArray($response['body']['documents.delete']); + $this->assertIsArray($response['body']['collections.create']); + $this->assertIsArray($response['body']['collections.read']); + $this->assertIsArray($response['body']['collections.update']); + $this->assertIsArray($response['body']['collections.delete']); + } + + + /** + * @depends testCreateCollection + */ + public function testGetCollectionUsage(array $data) + { + /** + * Test for FAILURE + */ + + $response = $this->client->call(Client::METHOD_GET, '/database/'.$data['moviesId'].'/usage', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), [ + 'range' => '32h' + ]); + + $this->assertEquals($response['headers']['status-code'], 400); + + $response = $this->client->call(Client::METHOD_GET, '/database/randomCollectionId/usage', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), [ + 'range' => '24h' + ]); + + $this->assertEquals($response['headers']['status-code'], 404); + + /** + * Test for SUCCESS + */ + $response = $this->client->call(Client::METHOD_GET, '/database/'.$data['moviesId'].'/usage', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), [ + 'range' => '24h' + ]); + + $this->assertEquals($response['headers']['status-code'], 200); + $this->assertEquals(count($response['body']), 6); + $this->assertEquals($response['body']['range'], '24h'); + $this->assertIsArray($response['body']['documents.count']); + $this->assertIsArray($response['body']['documents.create']); + $this->assertIsArray($response['body']['documents.read']); + $this->assertIsArray($response['body']['documents.update']); + $this->assertIsArray($response['body']['documents.delete']); + } +} \ No newline at end of file diff --git a/tests/e2e/Services/Database/DatabaseCustomServerTest.php b/tests/e2e/Services/Database/DatabaseCustomServerTest.php index e5a75fa68..eb68bb0dd 100644 --- a/tests/e2e/Services/Database/DatabaseCustomServerTest.php +++ b/tests/e2e/Services/Database/DatabaseCustomServerTest.php @@ -241,6 +241,214 @@ class DatabaseCustomServerTest extends Scope /** * @depends testDeleteIndex */ + public function testDeleteIndexOnDeleteAttribute($data) + { + $attribute1 = $this->client->call(Client::METHOD_POST, '/database/collections/' . $data['collectionId'] . '/attributes/string', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'attributeId' => 'attribute1', + 'size' => 16, + 'required' => true, + ]); + + $attribute2 = $this->client->call(Client::METHOD_POST, '/database/collections/' . $data['collectionId'] . '/attributes/string', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'attributeId' => 'attribute2', + 'size' => 16, + 'required' => true, + ]); + + $this->assertEquals(201, $attribute1['headers']['status-code']); + $this->assertEquals(201, $attribute2['headers']['status-code']); + $this->assertEquals('attribute1', $attribute1['body']['key']); + $this->assertEquals('attribute2', $attribute2['body']['key']); + + sleep(2); + + $index1 = $this->client->call(Client::METHOD_POST, '/database/collections/' . $data['collectionId'] . '/indexes', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'indexId' => 'index1', + 'type' => 'key', + 'attributes' => ['attribute1', 'attribute2'], + 'orders' => ['ASC', 'ASC'], + ]); + + $index2 = $this->client->call(Client::METHOD_POST, '/database/collections/' . $data['collectionId'] . '/indexes', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'indexId' => 'index2', + 'type' => 'key', + 'attributes' => ['attribute2'], + ]); + + $this->assertEquals(201, $index1['headers']['status-code']); + $this->assertEquals(201, $index2['headers']['status-code']); + $this->assertEquals('index1', $index1['body']['key']); + $this->assertEquals('index2', $index2['body']['key']); + + sleep(2); + + // Expected behavior: deleting attribute2 will cause index2 to be dropped, and index1 rebuilt with a single key + $deleted = $this->client->call(Client::METHOD_DELETE, '/database/collections/' . $data['collectionId'] . '/attributes/'. $attribute2['body']['key'], array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ])); + + $this->assertEquals($deleted['headers']['status-code'], 204); + + // wait for database worker to complete + sleep(2); + + $collection = $this->client->call(Client::METHOD_GET, '/database/collections/' . $data['collectionId'], array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ])); + + $this->assertEquals(200, $collection['headers']['status-code']); + $this->assertIsArray($collection['body']['indexes']); + $this->assertCount(1, $collection['body']['indexes']); + $this->assertEquals($index1['body']['key'], $collection['body']['indexes'][0]['key']); + $this->assertIsArray($collection['body']['indexes'][0]['attributes']); + $this->assertCount(1, $collection['body']['indexes'][0]['attributes']); + $this->assertEquals($attribute1['body']['key'], $collection['body']['indexes'][0]['attributes'][0]); + + // Delete attribute + $deleted = $this->client->call(Client::METHOD_DELETE, '/database/collections/' . $data['collectionId'] . '/attributes/' . $attribute1['body']['key'], array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ])); + + $this->assertEquals($deleted['headers']['status-code'], 204); + + return $data; + } + + public function testCleanupDuplicateIndexOnDeleteAttribute() + { + $collection = $this->client->call(Client::METHOD_POST, '/database/collections', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'collectionId' => 'unique()', + 'name' => 'TestCleanupDuplicateIndexOnDeleteAttribute', + 'read' => ['role:all'], + 'write' => ['role:all'], + 'permission' => 'document', + ]); + + $this->assertEquals(201, $collection['headers']['status-code']); + $this->assertNotEmpty($collection['body']['$id']); + + $collectionId = $collection['body']['$id']; + + $attribute1 = $this->client->call(Client::METHOD_POST, '/database/collections/' . $collectionId . '/attributes/string', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'attributeId' => 'attribute1', + 'size' => 16, + 'required' => true, + ]); + + $attribute2 = $this->client->call(Client::METHOD_POST, '/database/collections/' . $collectionId . '/attributes/string', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'attributeId' => 'attribute2', + 'size' => 16, + 'required' => true, + ]); + + $this->assertEquals(201, $attribute1['headers']['status-code']); + $this->assertEquals(201, $attribute2['headers']['status-code']); + $this->assertEquals('attribute1', $attribute1['body']['key']); + $this->assertEquals('attribute2', $attribute2['body']['key']); + + sleep(2); + + $index1 = $this->client->call(Client::METHOD_POST, '/database/collections/' . $collectionId . '/indexes', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'indexId' => 'index1', + 'type' => 'key', + 'attributes' => ['attribute1', 'attribute2'], + 'orders' => ['ASC', 'ASC'], + ]); + + $index2 = $this->client->call(Client::METHOD_POST, '/database/collections/' . $collectionId . '/indexes', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'indexId' => 'index2', + 'type' => 'key', + 'attributes' => ['attribute2'], + ]); + + $this->assertEquals(201, $index1['headers']['status-code']); + $this->assertEquals(201, $index2['headers']['status-code']); + $this->assertEquals('index1', $index1['body']['key']); + $this->assertEquals('index2', $index2['body']['key']); + + sleep(2); + + // Expected behavior: deleting attribute1 would cause index1 to be a duplicate of index2 and automatically removed + $deleted = $this->client->call(Client::METHOD_DELETE, '/database/collections/' . $collectionId . '/attributes/'. $attribute1['body']['key'], array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ])); + + $this->assertEquals($deleted['headers']['status-code'], 204); + + // wait for database worker to complete + sleep(2); + + $collection = $this->client->call(Client::METHOD_GET, '/database/collections/' . $collectionId, array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ])); + + $this->assertEquals(200, $collection['headers']['status-code']); + $this->assertIsArray($collection['body']['indexes']); + $this->assertCount(1, $collection['body']['indexes']); + $this->assertEquals($index2['body']['key'], $collection['body']['indexes'][0]['key']); + $this->assertIsArray($collection['body']['indexes'][0]['attributes']); + $this->assertCount(1, $collection['body']['indexes'][0]['attributes']); + $this->assertEquals($attribute2['body']['key'], $collection['body']['indexes'][0]['attributes'][0]); + + // Delete attribute + $deleted = $this->client->call(Client::METHOD_DELETE, '/database/collections/' . $collectionId . '/attributes/' . $attribute2['body']['key'], array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ])); + + $this->assertEquals($deleted['headers']['status-code'], 204); + } + + /** + * @depends testDeleteIndexOnDeleteAttribute + */ public function testDeleteCollection($data) { $collectionId = $data['collectionId']; @@ -307,6 +515,101 @@ class DatabaseCustomServerTest extends Scope $this->assertEquals($response['headers']['status-code'], 404); } + public function testAttributeCountLimit() + { + $collection = $this->client->call(Client::METHOD_POST, '/database/collections', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'collectionId' => 'unique()', + 'name' => 'attributeCountLimit', + 'read' => ['role:all'], + 'write' => ['role:all'], + 'permission' => 'document', + ]); + + $collectionId = $collection['body']['$id']; + + // load the collection up to the limit + for ($i=0; $i < 1012; $i++) { + $attribute = $this->client->call(Client::METHOD_POST, '/database/collections/' . $collectionId . '/attributes/integer', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'attributeId' => "attribute{$i}", + 'required' => false, + ]); + + $this->assertEquals(201, $attribute['headers']['status-code']); + } + + sleep(5); + + $tooMany = $this->client->call(Client::METHOD_POST, '/database/collections/' . $collectionId . '/attributes/integer', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'attributeId' => "tooMany", + 'required' => false, + ]); + + $this->assertEquals(400, $tooMany['headers']['status-code']); + $this->assertEquals('Attribute limit exceeded', $tooMany['body']['message']); + } + + public function testAttributeRowWidthLimit() + { + $collection = $this->client->call(Client::METHOD_POST, '/database/collections', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'collectionId' => 'attributeRowWidthLimit', + 'name' => 'attributeRowWidthLimit', + 'read' => ['role:all'], + 'write' => ['role:all'], + 'permission' => 'document', + ]); + + $this->assertEquals($collection['headers']['status-code'], 201); + $this->assertEquals($collection['body']['name'], 'attributeRowWidthLimit'); + + $collectionId = $collection['body']['$id']; + + // Add wide string attributes to approach row width limit + for ($i=0; $i < 15; $i++) { + $attribute = $this->client->call(Client::METHOD_POST, '/database/collections/' . $collectionId . '/attributes/string', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'attributeId' => "attribute{$i}", + 'size' => 1024, + 'required' => true, + ]); + + $this->assertEquals($attribute['headers']['status-code'], 201); + } + + sleep(5); + + $tooWide = $this->client->call(Client::METHOD_POST, '/database/collections/' . $collectionId . '/attributes/string', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'attributeId' => 'tooWide', + 'size' => 1024, + 'required' => true, + ]); + + $this->assertEquals(400, $tooWide['headers']['status-code']); + $this->assertEquals('Attribute limit exceeded', $tooWide['body']['message']); + } + public function testIndexLimitException() { $collection = $this->client->call(Client::METHOD_POST, '/database/collections', array_merge([ @@ -403,5 +706,13 @@ class DatabaseCustomServerTest extends Scope $this->assertEquals(400, $tooMany['headers']['status-code']); $this->assertEquals('Index limit exceeded', $tooMany['body']['message']); + + $collection = $this->client->call(Client::METHOD_DELETE, '/database/collections/' . $collectionId, array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ])); + + $this->assertEquals(204, $collection['headers']['status-code']); } } \ No newline at end of file diff --git a/tests/e2e/Services/Functions/FunctionsConsoleClientTest.php b/tests/e2e/Services/Functions/FunctionsConsoleClientTest.php new file mode 100644 index 000000000..ee15a81dc --- /dev/null +++ b/tests/e2e/Services/Functions/FunctionsConsoleClientTest.php @@ -0,0 +1,91 @@ +client->call(Client::METHOD_POST, '/functions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'functionId' => 'unique()', + 'name' => 'Test', + 'execute' => ['user:'.$this->getUser()['$id']], + 'runtime' => 'php-8.0', + 'vars' => [ + 'funcKey1' => 'funcValue1', + 'funcKey2' => 'funcValue2', + 'funcKey3' => 'funcValue3', + ], + 'events' => [ + 'account.create', + 'account.delete', + ], + 'schedule' => '0 0 1 1 *', + 'timeout' => 10, + ]); + + $this->assertEquals(201, $function['headers']['status-code']); + + return [ + 'functionId' => $function['body']['$id'] + ]; + } + + /** + * @depends testCreateFunction + */ + public function testGetCollectionUsage(array $data) + { + /** + * Test for FAILURE + */ + + $response = $this->client->call(Client::METHOD_GET, '/functions/'.$data['functionId'].'/usage', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), [ + 'range' => '232h' + ]); + + $this->assertEquals(400, $response['headers']['status-code']); + + $response = $this->client->call(Client::METHOD_GET, '/functions/randomFunctionId/usage', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), [ + 'range' => '24h' + ]); + + $this->assertEquals(404, $response['headers']['status-code']); + + /** + * Test for SUCCESS + */ + + $response = $this->client->call(Client::METHOD_GET, '/functions/'.$data['functionId'].'/usage', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), [ + 'range' => '24h' + ]); + + $this->assertEquals($response['headers']['status-code'], 200); + $this->assertEquals(count($response['body']), 4); + $this->assertEquals($response['body']['range'], '24h'); + $this->assertIsArray($response['body']['functions.executions']); + $this->assertIsArray($response['body']['functions.failures']); + $this->assertIsArray($response['body']['functions.compute']); + } + +} \ No newline at end of file diff --git a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php index cb820b45c..57b8ad921 100644 --- a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php +++ b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php @@ -78,6 +78,45 @@ class FunctionsCustomServerTest extends Scope * Test for SUCCESS */ + /** + * Test search queries + */ + $response = $this->client->call(Client::METHOD_GET, '/functions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'search' => $data['functionId'] + ]); + + $this->assertEquals($response['headers']['status-code'], 200); + $this->assertCount(1, $response['body']['functions']); + $this->assertEquals($response['body']['functions'][0]['name'], 'Test'); + + $response = $this->client->call(Client::METHOD_GET, '/functions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'search' => 'Test' + ]); + + $this->assertEquals($response['headers']['status-code'], 200); + $this->assertCount(1, $response['body']['functions']); + $this->assertEquals($response['body']['functions'][0]['$id'], $data['functionId']); + + $response = $this->client->call(Client::METHOD_GET, '/functions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'search' => 'php-8.0' + ]); + + $this->assertEquals($response['headers']['status-code'], 200); + $this->assertCount(1, $response['body']['functions']); + $this->assertEquals($response['body']['functions'][0]['$id'], $data['functionId']); + + /** + * Test pagination + */ $response = $this->client->call(Client::METHOD_POST, '/functions', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], @@ -122,7 +161,6 @@ class FunctionsCustomServerTest extends Scope $this->assertCount(1, $response['body']['functions']); $this->assertEquals($response['body']['functions'][0]['name'], 'Test 2'); - return $data; } @@ -283,6 +321,48 @@ class FunctionsCustomServerTest extends Scope $this->assertIsArray($function['body']['tags']); $this->assertCount(1, $function['body']['tags']); + /** + * Test search queries + */ + $function = $this->client->call(Client::METHOD_GET, '/functions/'.$data['functionId'].'/tags', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders(), [ + 'search' => $data['functionId'] + ])); + + $this->assertEquals($function['headers']['status-code'], 200); + $this->assertEquals($function['body']['sum'], 1); + $this->assertIsArray($function['body']['tags']); + $this->assertCount(1, $function['body']['tags']); + $this->assertEquals($function['body']['tags'][0]['$id'], $data['tagId']); + + $function = $this->client->call(Client::METHOD_GET, '/functions/'.$data['functionId'].'/tags', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders(), [ + 'search' => 'Test' + ])); + + $this->assertEquals($function['headers']['status-code'], 200); + $this->assertEquals($function['body']['sum'], 1); + $this->assertIsArray($function['body']['tags']); + $this->assertCount(1, $function['body']['tags']); + $this->assertEquals($function['body']['tags'][0]['$id'], $data['tagId']); + + $function = $this->client->call(Client::METHOD_GET, '/functions/'.$data['functionId'].'/tags', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders(), [ + 'search' => 'php-8.0' + ])); + + $this->assertEquals($function['headers']['status-code'], 200); + $this->assertEquals($function['body']['sum'], 1); + $this->assertIsArray($function['body']['tags']); + $this->assertCount(1, $function['body']['tags']); + $this->assertEquals($function['body']['tags'][0]['$id'], $data['tagId']); + return $data; } @@ -394,6 +474,36 @@ class FunctionsCustomServerTest extends Scope $this->assertCount(1, $function['body']['executions']); $this->assertEquals($function['body']['executions'][0]['$id'], $data['executionId']); + /** + * Test search queries + */ + + $response = $this->client->call(Client::METHOD_GET, '/functions/'.$data['functionId'].'/executions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'search' => $data['executionId'], + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals(1, $response['body']['sum']); + $this->assertIsInt($response['body']['sum']); + $this->assertCount(1, $response['body']['executions']); + $this->assertEquals($data['functionId'], $response['body']['executions'][0]['functionId']); + + $response = $this->client->call(Client::METHOD_GET, '/functions/'.$data['functionId'].'/executions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'search' => $data['functionId'], + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals(1, $response['body']['sum']); + $this->assertIsInt($response['body']['sum']); + $this->assertCount(1, $response['body']['executions']); + $this->assertEquals($data['executionId'], $response['body']['executions'][0]['$id']); + return $data; } diff --git a/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php b/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php index 4d83efb41..2572c7e09 100644 --- a/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php +++ b/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php @@ -87,6 +87,7 @@ class ProjectsConsoleClientTest extends Scope /** * Test for SUCCESS */ + $response = $this->client->call(Client::METHOD_GET, '/projects', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], @@ -97,6 +98,35 @@ class ProjectsConsoleClientTest extends Scope $this->assertEquals($id, $response['body']['projects'][0]['$id']); $this->assertEquals('Project Test', $response['body']['projects'][0]['name']); + /** + * Test search queries + */ + $response = $this->client->call(Client::METHOD_GET, '/projects', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders(), [ + 'search' => $id + ])); + + $this->assertEquals($response['headers']['status-code'], 200); + $this->assertEquals($response['body']['sum'], 1); + $this->assertIsArray($response['body']['projects']); + $this->assertCount(1, $response['body']['projects']); + $this->assertEquals($response['body']['projects'][0]['name'], 'Project Test'); + + $response = $this->client->call(Client::METHOD_GET, '/projects', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders(), [ + 'search' => 'Project Test' + ])); + + $this->assertEquals($response['headers']['status-code'], 200); + $this->assertEquals($response['body']['sum'], 1); + $this->assertIsArray($response['body']['projects']); + $this->assertCount(1, $response['body']['projects']); + $this->assertEquals($response['body']['projects'][0]['$id'], $data['projectId']); + /** * Test after pagination */ @@ -215,24 +245,16 @@ class ProjectsConsoleClientTest extends Scope ], $this->getHeaders())); $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals(count($response['body']), 8); $this->assertNotEmpty($response['body']); - $this->assertArrayHasKey('collections', $response['body']); - $this->assertArrayHasKey('documents', $response['body']); - $this->assertArrayHasKey('network', $response['body']); - $this->assertArrayHasKey('requests', $response['body']); - $this->assertArrayHasKey('storage', $response['body']); - $this->assertArrayHasKey('users', $response['body']); - $this->assertIsArray($response['body']['collections']['data']); - $this->assertIsInt($response['body']['collections']['total']); - $this->assertIsArray($response['body']['documents']['data']); - $this->assertIsInt($response['body']['documents']['total']); - $this->assertIsArray($response['body']['network']['data']); - $this->assertIsInt($response['body']['network']['total']); - $this->assertIsArray($response['body']['requests']['data']); - $this->assertIsInt($response['body']['requests']['total']); - $this->assertIsInt($response['body']['storage']['total']); - $this->assertIsArray($response['body']['users']['data']); - $this->assertIsInt($response['body']['users']['total']); + $this->assertEquals('30d', $response['body']['range']); + $this->assertIsArray($response['body']['requests']); + $this->assertIsArray($response['body']['network']); + $this->assertIsArray($response['body']['functions']); + $this->assertIsArray($response['body']['documents']); + $this->assertIsArray($response['body']['collections']); + $this->assertIsArray($response['body']['users']); + $this->assertIsArray($response['body']['storage']); /** * Test for FAILURE @@ -1430,6 +1452,17 @@ class ProjectsConsoleClientTest extends Scope /** * Test for FAILURE */ + $response = $this->client->call(Client::METHOD_PUT, '/projects/'.$id.'/platforms/error', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'name' => 'Flutter App (Android) 2', + 'key' => 'com.example.android2', + 'store' => '', + 'hostname' => '', + ]); + + $this->assertEquals(404, $response['headers']['status-code']); return $data; } diff --git a/tests/e2e/Services/Storage/StorageBase.php b/tests/e2e/Services/Storage/StorageBase.php index 57a5ee697..ab1a08c68 100644 --- a/tests/e2e/Services/Storage/StorageBase.php +++ b/tests/e2e/Services/Storage/StorageBase.php @@ -163,7 +163,7 @@ trait StorageBase 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), [ 'fileId' => 'unique()', - 'file' => new CURLFile(realpath(__DIR__ . '/../../../resources/logo.png'), 'image/png', 'logo.png'), + 'file' => new CURLFile(realpath(__DIR__ . '/../../../resources/file.png'), 'image/png', 'file.png'), 'read' => ['role:all'], 'write' => ['role:all'], ]); @@ -171,9 +171,9 @@ trait StorageBase $this->assertEquals($file['headers']['status-code'], 201); $this->assertNotEmpty($file['body']['$id']); $this->assertIsInt($file['body']['dateCreated']); - $this->assertEquals('logo.png', $file['body']['name']); - $this->assertEquals('image/png', $file['body']['mimeType']); - $this->assertEquals(47218, $file['body']['sizeOriginal']); + $this->assertEquals('file.png', $file['body']['name']); + $this->assertEquals('image/jpeg', $file['body']['mimeType']); + $this->assertEquals(16804, $file['body']['sizeOriginal']); $files = $this->client->call(Client::METHOD_GET, '/storage/files', array_merge([ 'content-type' => 'application/json', @@ -197,6 +197,32 @@ trait StorageBase $this->assertEquals($files['body']['files'][1]['$id'], $response['body']['files'][0]['$id']); $this->assertCount(1, $response['body']['files']); + $response = $this->client->call(Client::METHOD_GET, '/storage/files', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'search' => $data['fileId'], + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals(1, $response['body']['sum']); + $this->assertIsInt($response['body']['sum']); + $this->assertCount(1, $response['body']['files']); + $this->assertEquals('logo.png', $response['body']['files'][0]['name']); + + $response = $this->client->call(Client::METHOD_GET, '/storage/files', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'search' => 'logo', + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals(1, $response['body']['sum']); + $this->assertIsInt($response['body']['sum']); + $this->assertCount(1, $response['body']['files']); + $this->assertEquals($data['fileId'], $response['body']['files'][0]['$id']); + /** * Test for FAILURE */ diff --git a/tests/e2e/Services/Storage/StorageConsoleClientTest.php b/tests/e2e/Services/Storage/StorageConsoleClientTest.php index b2c958296..4ec1a6ce9 100644 --- a/tests/e2e/Services/Storage/StorageConsoleClientTest.php +++ b/tests/e2e/Services/Storage/StorageConsoleClientTest.php @@ -2,13 +2,91 @@ namespace Tests\E2E\Services\Storage; +use Tests\E2E\Client; use Tests\E2E\Scopes\Scope; -use Tests\E2E\Scopes\ProjectConsole; -use Tests\E2E\Scopes\SideClient; +use Tests\E2E\Scopes\ProjectCustom; +use Tests\E2E\Scopes\SideConsole; class StorageConsoleClientTest extends Scope { + use SideConsole; use StorageBase; - use ProjectConsole; - use SideClient; + use ProjectCustom; + + public function testGetStorageUsage() + { + /** + * Test for FAILURE + */ + $response = $this->client->call(Client::METHOD_GET, '/storage/usage', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), [ + 'range' => '32h' + ]); + + $this->assertEquals($response['headers']['status-code'], 400); + + /** + * Test for SUCCESS + */ + $response = $this->client->call(Client::METHOD_GET, '/storage/usage', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), [ + 'range' => '24h' + ]); + + $this->assertEquals($response['headers']['status-code'], 200); + $this->assertEquals(count($response['body']), 3); + $this->assertEquals($response['body']['range'], '24h'); + $this->assertIsArray($response['body']['storage']); + $this->assertIsArray($response['body']['files']); + } + + public function testGetStorageBucketUsage() + { + /** + * Test for FAILURE + */ + + $response = $this->client->call(Client::METHOD_GET, '/storage/default/usage', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), [ + 'range' => '32h' + ]); + + $this->assertEquals($response['headers']['status-code'], 400); + + // TODO: Uncomment once we implement check for missing bucketId in the usage endpoint. + + // $response = $this->client->call(Client::METHOD_GET, '/storage/randomBucketId/usage', array_merge([ + // 'content-type' => 'application/json', + // 'x-appwrite-project' => $this->getProject()['$id'] + // ], $this->getHeaders()), [ + // 'range' => '24h' + // ]); + + // $this->assertEquals($response['headers']['status-code'], 404); + + /** + * Test for SUCCESS + */ + $response = $this->client->call(Client::METHOD_GET, '/storage/default/usage', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), [ + 'range' => '24h' + ]); + + $this->assertEquals($response['headers']['status-code'], 200); + $this->assertEquals(count($response['body']), 6); + $this->assertEquals($response['body']['range'], '24h'); + $this->assertIsArray($response['body']['files.count']); + $this->assertIsArray($response['body']['files.create']); + $this->assertIsArray($response['body']['files.read']); + $this->assertIsArray($response['body']['files.update']); + $this->assertIsArray($response['body']['files.delete']); + } } \ No newline at end of file diff --git a/tests/e2e/Services/Teams/TeamsBase.php b/tests/e2e/Services/Teams/TeamsBase.php index 6c1ebbe36..b594bbfa6 100644 --- a/tests/e2e/Services/Teams/TeamsBase.php +++ b/tests/e2e/Services/Teams/TeamsBase.php @@ -172,6 +172,19 @@ trait TeamsBase $this->assertCount(1, $response['body']['teams']); $this->assertEquals('Manchester United', $response['body']['teams'][0]['name']); + $response = $this->client->call(Client::METHOD_GET, '/teams', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'search' => $data['teamUid'], + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertGreaterThan(0, $response['body']['sum']); + $this->assertIsInt($response['body']['sum']); + $this->assertCount(1, $response['body']['teams']); + $this->assertEquals('Arsenal', $response['body']['teams'][0]['name']); + $teams = $this->client->call(Client::METHOD_GET, '/teams', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], diff --git a/tests/e2e/Services/Users/UsersBase.php b/tests/e2e/Services/Users/UsersBase.php index 90b59a8c1..ee35e66fc 100644 --- a/tests/e2e/Services/Users/UsersBase.php +++ b/tests/e2e/Services/Users/UsersBase.php @@ -16,14 +16,14 @@ trait UsersBase 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), [ 'userId' => 'unique()', - 'email' => 'users.service@example.com', + 'email' => 'cristiano.ronaldo@manchester-united.co.uk', 'password' => 'password', - 'name' => 'Project User', + 'name' => 'Cristiano Ronaldo', ]); $this->assertEquals($user['headers']['status-code'], 201); - $this->assertEquals($user['body']['name'], 'Project User'); - $this->assertEquals($user['body']['email'], 'users.service@example.com'); + $this->assertEquals($user['body']['name'], 'Cristiano Ronaldo'); + $this->assertEquals($user['body']['email'], 'cristiano.ronaldo@manchester-united.co.uk'); $this->assertEquals($user['body']['status'], true); $this->assertGreaterThan(0, $user['body']['registration']); @@ -35,15 +35,15 @@ trait UsersBase 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), [ 'userId' => 'user1', - 'email' => 'users.service1@example.com', + 'email' => 'lionel.messi@psg.fr', 'password' => 'password', - 'name' => 'Project User', + 'name' => 'Lionel Messi', ]); $this->assertEquals($res['headers']['status-code'], 201); $this->assertEquals($res['body']['$id'], 'user1'); - $this->assertEquals($res['body']['name'], 'Project User'); - $this->assertEquals($res['body']['email'], 'users.service1@example.com'); + $this->assertEquals($res['body']['name'], 'Lionel Messi'); + $this->assertEquals($res['body']['email'], 'lionel.messi@psg.fr'); $this->assertEquals(true, $res['body']['status']); $this->assertGreaterThan(0, $res['body']['registration']); @@ -56,7 +56,7 @@ trait UsersBase public function testListUsers(array $data): void { /** - * Test for SUCCESS + * Test for SUCCESS listUsers */ $response = $this->client->call(Client::METHOD_GET, '/users', array_merge([ 'content-type' => 'application/json', @@ -82,8 +82,72 @@ trait UsersBase $this->assertNotEmpty($response['body']); $this->assertNotEmpty($response['body']['users']); $this->assertCount(1, $response['body']['users']); - $this->assertEquals($response['body']['users'][0]['$id'], 'user1'); + + /** + * Test for SUCCESS searchUsers + */ + $response = $this->client->call(Client::METHOD_GET, '/users', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'search' => 'Ronaldo' + ]); + $this->assertEquals($response['headers']['status-code'], 200); + $this->assertNotEmpty($response['body']); + $this->assertNotEmpty($response['body']['users']); + $this->assertCount(1, $response['body']['users']); + $this->assertEquals($response['body']['users'][0]['$id'], $data['userId']); + + $response = $this->client->call(Client::METHOD_GET, '/users', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'search' => 'cristiano.ronaldo' + ]); + $this->assertEquals($response['headers']['status-code'], 200); + $this->assertNotEmpty($response['body']); + $this->assertNotEmpty($response['body']['users']); + $this->assertCount(1, $response['body']['users']); + $this->assertEquals($response['body']['users'][0]['$id'], $data['userId']); + + $response = $this->client->call(Client::METHOD_GET, '/users', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'search' => 'manchester' + ]); + $this->assertEquals($response['headers']['status-code'], 200); + $this->assertNotEmpty($response['body']); + $this->assertNotEmpty($response['body']['users']); + $this->assertCount(1, $response['body']['users']); + $this->assertEquals($response['body']['users'][0]['$id'], $data['userId']); + + $response = $this->client->call(Client::METHOD_GET, '/users', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'search' => 'manchester-united.co.uk' + ]); + + $this->assertEquals($response['headers']['status-code'], 200); + $this->assertIsArray($response['body']); + $this->assertIsArray($response['body']['users']); + $this->assertIsInt($response['body']['sum']); + $this->assertEquals(1, $response['body']['sum']); + $this->assertCount(1, $response['body']['users']); + + $response = $this->client->call(Client::METHOD_GET, '/users', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'search' => $data['userId'] + ]); + $this->assertEquals($response['headers']['status-code'], 200); + $this->assertNotEmpty($response['body']); + $this->assertNotEmpty($response['body']['users']); + $this->assertCount(1, $response['body']['users']); + $this->assertEquals($response['body']['users'][0]['$id'], $data['userId']); } /** @@ -100,8 +164,8 @@ trait UsersBase ], $this->getHeaders())); $this->assertEquals($user['headers']['status-code'], 200); - $this->assertEquals($user['body']['name'], 'Project User'); - $this->assertEquals($user['body']['email'], 'users.service@example.com'); + $this->assertEquals($user['body']['name'], 'Cristiano Ronaldo'); + $this->assertEquals($user['body']['email'], 'cristiano.ronaldo@manchester-united.co.uk'); $this->assertEquals($user['body']['status'], true); $this->assertGreaterThan(0, $user['body']['registration']); @@ -132,21 +196,6 @@ trait UsersBase $this->assertIsInt($users['body']['sum']); $this->assertGreaterThan(0, $users['body']['sum']); - $users = $this->client->call(Client::METHOD_GET, '/users', array_merge([ - 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getProject()['$id'], - ], $this->getHeaders()), [ - 'search' => 'example.com' - ]); - - $this->assertEquals($users['headers']['status-code'], 200); - $this->assertIsArray($users['body']); - $this->assertIsArray($users['body']['users']); - $this->assertIsInt($users['body']['sum']); - $this->assertEquals(2, $users['body']['sum']); - $this->assertGreaterThan(0, $users['body']['sum']); - $this->assertCount(2, $users['body']['users']); - return $data; } diff --git a/tests/e2e/Services/Users/UsersConsoleClientTest.php b/tests/e2e/Services/Users/UsersConsoleClientTest.php new file mode 100644 index 000000000..90a10a0c4 --- /dev/null +++ b/tests/e2e/Services/Users/UsersConsoleClientTest.php @@ -0,0 +1,83 @@ +client->call(Client::METHOD_GET, '/users/usage', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), [ + 'range' => '32h', + 'provider' => 'email' + ]); + + $this->assertEquals($response['headers']['status-code'], 400); + + $response = $this->client->call(Client::METHOD_GET, '/users/usage', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), [ + 'range' => '24h', + 'provider' => 'some-random-provider' + ]); + + $this->assertEquals($response['headers']['status-code'], 400); + + /** + * Test for SUCCESS + */ + $response = $this->client->call(Client::METHOD_GET, '/users/usage', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), [ + 'range' => '24h', + 'provider' => 'email' + ]); + + $this->assertEquals($response['headers']['status-code'], 200); + $this->assertEquals(count($response['body']), 9); + $this->assertEquals($response['body']['range'], '24h'); + $this->assertIsArray($response['body']['users.count']); + $this->assertIsArray($response['body']['users.create']); + $this->assertIsArray($response['body']['users.read']); + $this->assertIsArray($response['body']['users.update']); + $this->assertIsArray($response['body']['users.delete']); + $this->assertIsArray($response['body']['sessions.create']); + $this->assertIsArray($response['body']['sessions.provider.create']); + $this->assertIsArray($response['body']['sessions.delete']); + + $response = $this->client->call(Client::METHOD_GET, '/users/usage', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), [ + 'range' => '24h' + ]); + + $this->assertEquals($response['headers']['status-code'], 200); + $this->assertEquals(count($response['body']), 9); + $this->assertEquals($response['body']['range'], '24h'); + $this->assertIsArray($response['body']['users.count']); + $this->assertIsArray($response['body']['users.create']); + $this->assertIsArray($response['body']['users.read']); + $this->assertIsArray($response['body']['users.update']); + $this->assertIsArray($response['body']['users.delete']); + $this->assertIsArray($response['body']['sessions.create']); + $this->assertIsArray($response['body']['sessions.provider.create']); + $this->assertIsArray($response['body']['sessions.delete']); + } +} \ No newline at end of file diff --git a/tests/e2e/Services/Users/UsersCustomServerTest.php b/tests/e2e/Services/Users/UsersCustomServerTest.php index c5e4ff8c1..cee654e6c 100644 --- a/tests/e2e/Services/Users/UsersCustomServerTest.php +++ b/tests/e2e/Services/Users/UsersCustomServerTest.php @@ -2,6 +2,7 @@ namespace Tests\E2E\Services\Users; +use Tests\E2E\Client; use Tests\E2E\Scopes\ProjectCustom; use Tests\E2E\Scopes\Scope; use Tests\E2E\Scopes\SideServer; @@ -11,4 +12,49 @@ class UsersCustomServerTest extends Scope use UsersBase; use ProjectCustom; use SideServer; + + public function testDeprecatedUsers():array + { + /** + * Test for FAILURE (don't allow recreating account with same custom ID) + */ + + // Create user with custom ID 'meldiron' + $response = $this->client->call(Client::METHOD_POST, '/users', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'userId' => 'meldiron', + 'email' => 'matej@appwrite.io', + 'password' => 'my-superstr0ng-password', + 'name' => 'Matej Bačo' + ]); + + $this->assertEquals(201, $response['headers']['status-code']); + + // Delete user with custom ID 'meldiron' + $response = $this->client->call(Client::METHOD_DELETE, '/users/meldiron', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(204, $response['headers']['status-code']); + + // Try to create user with custom ID 'meldiron' again, but now it should fail + $response1 = $this->client->call(Client::METHOD_POST, '/users', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'userId' => 'meldiron', + 'email' => 'matej2@appwrite.io', + 'password' => 'someones-superstr0ng-password', + 'name' => 'Matej Bačo Second' + ]); + + $this->assertEquals(409, $response1['headers']['status-code']); + $this->assertEquals('Account already exists', $response1['body']['message']); + + return []; + } + } \ No newline at end of file