CodeMirror/appwrite
1
0
Fork 0
mirror of synced 2024-05-17 11:12:41 +12:00
Code Issues Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/feat-db-pools' into feat-db-pools-master-sync

Browse source
This commit is contained in:
Damodar Lohani 2023-02-16 00:44:29 +00:00
parent 9fcee36cd2 1832c3c15f
commit 5acd78463a
91 changed files with 1986 additions and 4994 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
.env
View file

@ -36,10 +36,6 @@ _APP_CONNECTIONS_STORAGE=local://localhost
_APP_STORAGE_ANTIVIRUS=disabled
_APP_STORAGE_ANTIVIRUS_HOST=clamav
_APP_STORAGE_ANTIVIRUS_PORT=3310
_APP_INFLUXDB_HOST=influxdb
_APP_INFLUXDB_PORT=8086
_APP_STATSD_HOST=telegraf
_APP_STATSD_PORT=8125
_APP_SMTP_HOST=maildev
_APP_SMTP_PORT=1025
_APP_SMTP_SECURE=

2
.github/workflows/tests.yml vendored
View file

@ -33,7 +33,7 @@ jobs:
docker compose build appwrite
docker compose up -d
sleep 30
- name: Doctor
run: docker compose exec -T appwrite doctor

34
CHANGES.md
View file

@ -31,6 +31,40 @@
- Get default region from environment on project create [#4780](https://github.com/appwrite/appwrite/pull/4780)
- Fix french translation [#4782](https://github.com/appwrite/appwrite/pull/4782)
- Fix max mimetype size [#4814](https://github.com/appwrite/appwrite/pull/4814)
- New usage metrics collection flow [#4770](https://github.com/appwrite/appwrite/pull/4770)
- Deprecated influxdb, telegraf containers and removed all of their occurrences from the code.
- Removed _APP_INFLUXDB_HOST, _APP_INFLUXDB_PORT, _APP_STATSD_HOST, _APP_STATSD_PORT env variables.
- Removed usage labels dependency.
- Dropped type attribute from stats collection.
- Usage metrics are processed via new usage worker.
- Metrics changes:
- Storage
- deprecated
- filesCreate, filesRead, filesUpdate, filesDelete, bucketsCreate, bucketsRead, bucketsUpdate, bucketsDelete.
- renamed
- filesCount to filesTotal, storage to filesStorage, bucketsCount to bucketsTotal.
- Auth
- deprecated
- usersCreate, usersRead, usersUpdate, usersDelete, sessionsCreate sessionsProviderCreate, sessionsDelete.
- renamed
- usersCount to usersTotal.
- added
- sessionsTotal.
- Databases
- deprecated
- databasesCreate, databasesRead, databasesDelete, documentsCreate, documentsRead, documentsUpdate, documentsDelete, collectionsCreate, collectionsRead, collectionsUpdate, collectionsDelete.
- renamed
- databasesCount to databasesTotal, collectionsCount to collectionsTotal, documentsCount to documentsTotal.
- Functions
- deprecated
- executionsFailure, executionsSuccess, buildsFailure, buildsSuccess, executionsFailure, executionsSuccess.
- renamed
- executionsTime to executionsCompute, buildsTime to buildsCompute, documentsCount to documentsTotal.
- added
- functionsTotal, buildsStorage, deploymentsTotal, deploymentsStorage.
- Project
- renamed
- executions to executionsTotal, builds to buildsTotal, requests to requestsTotal, storage to filesStorage, buckets to bucketsTotal, users to usersTotal, documents to documentsTotal, collections to collectionsTotal, databases to databasesTotal.
## Bugs
- Fix invited account verified status [#4776](https://github.com/appwrite/appwrite/pull/4776)

2
CONTRIBUTING.md
View file

@ -238,8 +238,6 @@ Appwrite stack is a combination of a variety of open-source technologies and too
- Redis - for managing cache and in-memory data (currently, we do not use Redis for persistent data).
- MariaDB - for database storage and queries.
- InfluxDB - for managing stats and time-series based data.
- Statsd - for sending data over UDP protocol (using Telegraf).
- ClamAV - for validating and scanning storage files.
- Imagemagick - for manipulating and managing image media files.
- Webp - for better compression of images on supporting clients.

8
Dockerfile
View file

@ -62,10 +62,6 @@ ENV _APP_SERVER=swoole \
_APP_DB_USER=root \
_APP_DB_PASS=password \
_APP_DB_SCHEMA=appwrite \
_APP_INFLUXDB_HOST=influxdb \
_APP_INFLUXDB_PORT=8086 \
_APP_STATSD_HOST=telegraf \
_APP_STATSD_PORT=8125 \
_APP_FUNCTIONS_SIZE_LIMIT=30000000 \
_APP_FUNCTIONS_TIMEOUT=900 \
_APP_FUNCTIONS_CPUS=1 \
@ -118,7 +114,6 @@ RUN chmod +x /usr/local/bin/doctor && \
chmod +x /usr/local/bin/patch-create-missing-schedules && \
chmod +x /usr/local/bin/maintenance && \
chmod +x /usr/local/bin/volume-sync && \
chmod +x /usr/local/bin/usage && \
chmod +x /usr/local/bin/install && \
chmod +x /usr/local/bin/migrate && \
chmod +x /usr/local/bin/realtime && \
@ -136,7 +131,8 @@ RUN chmod +x /usr/local/bin/doctor && \
chmod +x /usr/local/bin/worker-builds && \
chmod +x /usr/local/bin/worker-mails && \
chmod +x /usr/local/bin/worker-messaging && \
chmod +x /usr/local/bin/worker-webhooks
chmod +x /usr/local/bin/worker-webhooks && \
chmod +x /usr/local/bin/worker-usage
# Letsencrypt Permissions
RUN mkdir -p /etc/letsencrypt/live/ && chmod -Rf 755 /etc/letsencrypt/live/

23
app/cli.php
View file

@ -116,29 +116,6 @@ CLI::setResource('getProjectDB', function (Group $pools, Database $dbForConsole,
return $getProjectDB;
}, ['pools', 'dbForConsole', 'cache']);
CLI::setResource('influxdb', function (Registry $register) {
$client = $register->get('influxdb'); /** @var InfluxDB\Client $client */
$attempts = 0;
$max = 10;
$sleep = 1;
do { // check if telegraf database is ready
try {
$attempts++;
$database = $client->selectDB('telegraf');
if (in_array('telegraf', $client->listDatabases())) {
break; // leave the do-while if successful
}
} catch (\Throwable $th) {
Console::warning("InfluxDB not ready. Retrying connection ({$attempts})...");
if ($attempts >= $max) {
throw new \Exception('InfluxDB database not ready yet');
}
sleep($sleep);
}
} while ($attempts < $max);
return $database;
}, ['register']);
CLI::setResource('queueForFunctions', function (Group $pools) {
return new Func($pools->get('queue')->pop()->getResource());

99
app/config/collections.php
View file

@ -192,7 +192,7 @@ $collections = [
'format' => '',
'size' => Database::LENGTH_KEY,
'signed' => true,
'required' => false,
'required' => true,
'default' => null,
'array' => false,
'filters' => [],
@ -214,7 +214,7 @@ $collections = [
'format' => '',
'size' => Database::LENGTH_KEY,
'signed' => true,
'required' => false,
'required' => true,
'default' => null,
'array' => false,
'filters' => [],
@ -369,7 +369,7 @@ $collections = [
'format' => '',
'size' => Database::LENGTH_KEY,
'signed' => true,
'required' => false,
'required' => true,
'default' => null,
'array' => false,
'filters' => [],
@ -391,7 +391,7 @@ $collections = [
'format' => '',
'size' => Database::LENGTH_KEY,
'signed' => true,
'required' => false,
'required' => true,
'default' => null,
'array' => false,
'filters' => [],
@ -496,7 +496,7 @@ $collections = [
'format' => '',
'size' => Database::LENGTH_KEY,
'signed' => true,
'required' => false,
'required' => true,
'default' => null,
'array' => false,
'filters' => [],
@ -791,7 +791,7 @@ $collections = [
'format' => '',
'size' => Database::LENGTH_KEY,
'signed' => true,
'required' => false,
'required' => true,
'default' => null,
'array' => false,
'filters' => [],
@ -892,7 +892,7 @@ $collections = [
'format' => '',
'size' => Database::LENGTH_KEY,
'signed' => true,
'required' => false,
'required' => true,
'default' => null,
'array' => false,
'filters' => [],
@ -986,7 +986,7 @@ $collections = [
'format' => '',
'size' => Database::LENGTH_KEY,
'signed' => true,
'required' => false,
'required' => true,
'default' => null,
'array' => false,
'filters' => [],
@ -1091,7 +1091,7 @@ $collections = [
'format' => '',
'size' => Database::LENGTH_KEY,
'signed' => true,
'required' => false,
'required' => true,
'default' => null,
'array' => false,
'filters' => [],
@ -1203,7 +1203,7 @@ $collections = [
'format' => '',
'size' => Database::LENGTH_KEY,
'signed' => true,
'required' => false,
'required' => true,
'default' => null,
'array' => false,
'filters' => [],
@ -1579,7 +1579,7 @@ $collections = [
'format' => '',
'size' => Database::LENGTH_KEY,
'signed' => true,
'required' => false,
'required' => true,
'default' => null,
'array' => false,
'filters' => [],
@ -1673,7 +1673,7 @@ $collections = [
'format' => '',
'size' => Database::LENGTH_KEY,
'signed' => true,
'required' => false,
'required' => true,
'default' => null,
'array' => false,
'filters' => [],
@ -2014,7 +2014,7 @@ $collections = [
'format' => '',
'size' => Database::LENGTH_KEY,
'signed' => true,
'required' => false,
'required' => true,
'default' => null,
'array' => false,
'filters' => [],
@ -2036,7 +2036,7 @@ $collections = [
'format' => '',
'size' => Database::LENGTH_KEY,
'signed' => true,
'required' => false,
'required' => true,
'default' => null,
'array' => false,
'filters' => [],
@ -2240,7 +2240,7 @@ $collections = [
'format' => '',
'size' => Database::LENGTH_KEY,
'signed' => true,
'required' => false,
'required' => true,
'default' => null,
'array' => false,
'filters' => [],
@ -2284,7 +2284,7 @@ $collections = [
'format' => '',
'size' => Database::LENGTH_KEY,
'signed' => true,
'required' => false,
'required' => true,
'default' => null,
'array' => false,
'filters' => [],
@ -2409,7 +2409,7 @@ $collections = [
'format' => '',
'size' => Database::LENGTH_KEY,
'signed' => true,
'required' => false,
'required' => true,
'default' => null,
'array' => false,
'filters' => [],
@ -2442,7 +2442,7 @@ $collections = [
'format' => '',
'size' => Database::LENGTH_KEY,
'signed' => true,
'required' => false,
'required' => true,
'default' => null,
'array' => false,
'filters' => [],
@ -2633,7 +2633,7 @@ $collections = [
'format' => '',
'size' => Database::LENGTH_KEY,
'signed' => true,
'required' => false,
'required' => true,
'default' => null,
'array' => false,
'filters' => [],
@ -2760,7 +2760,7 @@ $collections = [
'format' => '',
'size' => Database::LENGTH_KEY,
'signed' => true,
'required' => false,
'required' => true,
'default' => null,
'array' => false,
'filters' => [],
@ -2782,7 +2782,7 @@ $collections = [
'format' => '',
'size' => Database::LENGTH_KEY,
'signed' => true,
'required' => false,
'required' => true,
'default' => null,
'array' => false,
'filters' => [],
@ -3208,7 +3208,7 @@ $collections = [
'type' => Database::VAR_INTEGER,
'format' => '',
'size' => 8,
'signed' => false,
'signed' => true,
'required' => true,
'default' => null,
'array' => false,
@ -3236,17 +3236,6 @@ $collections = [
'array' => false,
'filters' => [],
],
[
'$id' => ID::custom('type'),
'type' => Database::VAR_INTEGER,
'format' => '',
'size' => 1,
'signed' => false,
'required' => true,
'default' => 0, // 0 -> count, 1 -> sum
'array' => false,
'filters' => [],
],
],
'indexes' => [
[
@ -3265,14 +3254,51 @@ $collections = [
],
[
'$id' => ID::custom('_key_metric_period_time'),
'type' => Database::INDEX_KEY,
'type' => Database::INDEX_UNIQUE,
'attributes' => ['metric', 'period', 'time'],
'lengths' => [],
'orders' => [Database::ORDER_DESC],
],
],
],
'statsLogger' => [
'$collection' => ID::custom(Database::METADATA),
'$id' => ID::custom('statsLogger'),
'name' => 'StatsLogger',
'attributes' => [
[
'$id' => ID::custom('time'),
'type' => Database::VAR_DATETIME,
'format' => '',
'size' => 0,
'signed' => false,
'required' => false,
'default' => null,
'array' => false,
'filters' => ['datetime'],
],
[
'$id' => ID::custom('metrics'),
'type' => Database::VAR_STRING,
'format' => '',
'size' => 5012,
'signed' => true,
'required' => false,
'default' => [],
'array' => false,
'filters' => ['json'],
],
],
'indexes' => [
[
'$id' => ID::custom('_key_time'),
'type' => Database::INDEX_KEY,
'attributes' => ['time'],
'lengths' => [],
'orders' => [Database::ORDER_DESC],
],
],
],
'realtime' => [
'$collection' => ID::custom(Database::METADATA),
'$id' => ID::custom('realtime'),
@ -3322,6 +3348,7 @@ $collections = [
],
]
],
'cache' => [
'$collection' => Database::METADATA,
'$id' => 'cache',
@ -3642,7 +3669,7 @@ $collections = [
'format' => '',
'size' => Database::LENGTH_KEY,
'signed' => true,
'required' => false,
'required' => true,
'default' => null,
'array' => false,
'filters' => [],

2
app/config/specs/open-api3-latest-client.json
View file

File diff suppressed because one or more lines are too long

2
app/config/specs/open-api3-latest-console.json
View file

File diff suppressed because one or more lines are too long

2
app/config/specs/open-api3-latest-server.json
View file

File diff suppressed because one or more lines are too long

2
app/config/specs/swagger2-latest-client.json
View file

File diff suppressed because one or more lines are too long

2
app/config/specs/swagger2-latest-console.json
View file

File diff suppressed because one or more lines are too long

2
app/config/specs/swagger2-latest-server.json
View file

File diff suppressed because one or more lines are too long

24
app/config/usage.php Normal file
View file

@ -0,0 +1,24 @@
<?php
return [
'24h' => [
'period' => '1h',
'limit' => 24,
'factor' => 3600,
],
'7d' => [
'period' => '1d',
'limit' => 7,
'factor' => 86400,
],
'30d' => [
'period' => '1d',
'limit' => 30,
'factor' => 86400,
],
'90d' => [
'period' => '1d',
'limit' => 90,
'factor' => 86400,
],
];

50
app/config/variables.php
View file

@ -143,7 +143,7 @@ return [
],
[
'name' => '_APP_USAGE_STATS',
'description' => 'This variable allows you to disable the collection and displaying of usage stats. This value is set to \'enabled\' by default, to disable the usage stats set the value to \'disabled\'. When disabled, it\'s recommended to turn off the Worker Usage, Influxdb and Telegraf containers for better resource usage.',
'description' => 'This variable allows you to disable the collection and displaying of usage stats. This value is set to \'enabled\' by default, to disable the usage stats set the value to \'disabled\'. When disabled, it\'s recommended to turn off the Worker Usage container for better resource usage.',
'introduction' => '0.7.0',
'default' => 'enabled',
'required' => false,
@ -344,54 +344,6 @@ return [
// ]
],
],
[
'category' => 'InfluxDB',
'description' => 'Appwrite uses an InfluxDB server for managing time-series data and server stats. The InfluxDB env vars are used to allow Appwrite server to connect to the InfluxDB container.',
'variables' => [
[
'name' => '_APP_INFLUXDB_HOST',
'description' => 'InfluxDB server host name address. Default value is: \'influxdb\'.',
'introduction' => '',
'default' => 'influxdb',
'required' => false,
'question' => '',
'filter' => ''
],
[
'name' => '_APP_INFLUXDB_PORT',
'description' => 'InfluxDB server TCP port. Default value is: \'8086\'.',
'introduction' => '',
'default' => '8086',
'required' => false,
'question' => '',
'filter' => ''
],
],
],
[
'category' => 'StatsD',
'description' => 'Appwrite uses a StatsD server for aggregating and sending stats data over a fast UDP connection. The StatsD env vars are used to allow Appwrite server to connect to the StatsD container.',
'variables' => [
[
'name' => '_APP_STATSD_HOST',
'description' => 'StatsD server host name address. Default value is: \'telegraf\'.',
'introduction' => '',
'default' => 'telegraf',
'required' => false,
'question' => '',
'filter' => ''
],
[
'name' => '_APP_STATSD_PORT',
'description' => 'StatsD server TCP port. Default value is: \'8125\'.',
'introduction' => '',
'default' => '8125',
'required' => false,
'question' => '',
'filter' => ''
],
],
],
[
'category' => 'SMTP',
'description' => "Appwrite is using an SMTP server for emailing your projects users and server admins. The SMTP env vars are used to allow Appwrite server to connect to the SMTP container.\n\nIf running in production, it might be easier to use a 3rd party SMTP server as it might be a little more difficult to set up a production SMTP server that will not send all your emails into your user\'s SPAM folder.",

32
app/controllers/api/account.php
View file

@ -53,7 +53,6 @@ App::post('/v1/account/invite')
->label('audits.event', 'user.create')
->label('audits.resource', 'user/{response.$id}')
->label('audits.userId', '{response.$id}')
->label('usage.metric', 'users.{scope}.requests.create')
->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
->label('sdk.namespace', 'account')
->label('sdk.method', 'createWithInviteCode')
@ -145,7 +144,6 @@ App::post('/v1/account')
->label('audits.event', 'user.create')
->label('audits.resource', 'user/{response.$id}')
->label('audits.userId', '{response.$id}')
->label('usage.metric', 'users.{scope}.requests.create')
->label('sdk.auth', [])
->label('sdk.namespace', 'account')
->label('sdk.method', 'create')
@ -239,8 +237,6 @@ App::post('/v1/account/sessions/email')
->label('audits.event', 'session.create')
->label('audits.resource', 'user/{response.userId}')
->label('audits.userId', '{response.userId}')
->label('usage.metric', 'sessions.{scope}.requests.create')
->label('usage.params', ['provider:email'])
->label('sdk.auth', [])
->label('sdk.namespace', 'account')
->label('sdk.method', 'createEmailSession')
@ -467,8 +463,6 @@ App::get('/v1/account/sessions/oauth2/:provider/redirect')
->label('abuse-limit', 50)
->label('abuse-key', 'ip:{ip}')
->label('docs', false)
->label('usage.metric', 'sessions.{scope}.requests.create')
->label('usage.params', ['provider:{request.provider}'])
->param('provider', '', new WhiteList(\array_keys(Config::getParam('providers')), true), 'OAuth2 provider.')
->param('code', '', new Text(2048), 'OAuth2 code.')
->param('state', '', new Text(2048), 'OAuth2 state params.', true)
@ -860,8 +854,6 @@ App::put('/v1/account/sessions/magic-url')
->label('audits.event', 'session.update')
->label('audits.resource', 'user/{response.userId}')
->label('audits.userId', '{response.userId}')
->label('usage.metric', 'sessions.{scope}.requests.create')
->label('usage.params', ['provider:magic-url'])
->label('sdk.auth', [])
->label('sdk.namespace', 'account')
->label('sdk.method', 'updateMagicURLSession')
@ -1099,8 +1091,6 @@ App::put('/v1/account/sessions/phone')
->groups(['api', 'account', 'session'])
->label('scope', 'public')
->label('event', 'users.[userId].sessions.[sessionId].create')
->label('usage.metric', 'sessions.{scope}.requests.create')
->label('usage.params', ['provider:phone'])
->label('sdk.auth', [])
->label('sdk.namespace', 'account')
->label('sdk.method', 'updatePhoneSession')
@ -1218,8 +1208,6 @@ App::post('/v1/account/sessions/anonymous')
->label('audits.event', 'session.create')
->label('audits.resource', 'user/{response.userId}')
->label('audits.userId', '{response.userId}')
->label('usage.metric', 'sessions.{scope}.requests.create')
->label('usage.params', ['provider:anonymous'])
->label('sdk.auth', [])
->label('sdk.namespace', 'account')
->label('sdk.method', 'createAnonymousSession')
@ -1394,7 +1382,6 @@ App::get('/v1/account')
->desc('Get Account')
->groups(['api', 'account'])
->label('scope', 'account')
->label('usage.metric', 'users.{scope}.requests.read')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'get')
@ -1413,7 +1400,6 @@ App::get('/v1/account/prefs')
->desc('Get Account Preferences')
->groups(['api', 'account'])
->label('scope', 'account')
->label('usage.metric', 'users.{scope}.requests.read')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'getPrefs')
@ -1434,7 +1420,6 @@ App::get('/v1/account/sessions')
->desc('List Sessions')
->groups(['api', 'account'])
->label('scope', 'account')
->label('usage.metric', 'users.{scope}.requests.read')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'listSessions')
@ -1471,7 +1456,6 @@ App::get('/v1/account/logs')
->desc('List Logs')
->groups(['api', 'account'])
->label('scope', 'account')
->label('usage.metric', 'users.{scope}.requests.read')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'listLogs')
@ -1532,7 +1516,6 @@ App::get('/v1/account/sessions/:sessionId')
->desc('Get Session')
->groups(['api', 'account'])
->label('scope', 'account')
->label('usage.metric', 'users.{scope}.requests.read')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'getSession')
@ -1578,7 +1561,6 @@ App::patch('/v1/account/name')
->label('scope', 'account')
->label('audits.event', 'user.update')
->label('audits.resource', 'user/{response.$id}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'updateName')
@ -1610,7 +1592,6 @@ App::patch('/v1/account/password')
->label('audits.event', 'user.update')
->label('audits.resource', 'user/{response.$id}')
->label('audits.userId', '{response.$id}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'updatePassword')
@ -1649,7 +1630,6 @@ App::patch('/v1/account/email')
->label('scope', 'account')
->label('audits.event', 'user.update')
->label('audits.resource', 'user/{response.$id}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'updateEmail')
@ -1701,7 +1681,6 @@ App::patch('/v1/account/phone')
->label('scope', 'account')
->label('audits.event', 'user.update')
->label('audits.resource', 'user/{response.$id}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'updatePhone')
@ -1749,7 +1728,6 @@ App::patch('/v1/account/prefs')
->label('scope', 'account')
->label('audits.event', 'user.update')
->label('audits.resource', 'user/{response.$id}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'updatePrefs')
@ -1778,7 +1756,6 @@ App::patch('/v1/account/status')
->label('scope', 'account')
->label('audits.event', 'user.update')
->label('audits.resource', 'user/{response.$id}')
->label('usage.metric', 'users.{scope}.requests.delete')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'updateStatus')
@ -1813,7 +1790,6 @@ App::delete('/v1/account/sessions/:sessionId')
->label('event', 'users.[userId].sessions.[sessionId].delete')
->label('audits.event', 'session.delete')
->label('audits.resource', 'user/{user.$id}')
->label('usage.metric', 'sessions.{scope}.requests.delete')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'deleteSession')
@ -1887,7 +1863,6 @@ App::patch('/v1/account/sessions/:sessionId')
->label('audits.event', 'session.update')
->label('audits.resource', 'user/{response.userId}')
->label('audits.userId', '{response.userId}')
->label('usage.metric', 'sessions.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'updateSession')
@ -1972,7 +1947,6 @@ App::delete('/v1/account/sessions')
->label('event', 'users.[userId].sessions.[sessionId].delete')
->label('audits.event', 'session.delete')
->label('audits.resource', 'user/{user.$id}')
->label('usage.metric', 'sessions.{scope}.requests.delete')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'deleteSessions')
@ -2034,7 +2008,6 @@ App::post('/v1/account/recovery')
->label('audits.event', 'recovery.create')
->label('audits.resource', 'user/{response.userId}')
->label('audits.userId', '{response.userId}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'createRecovery')
@ -2164,7 +2137,6 @@ App::put('/v1/account/recovery')
->label('audits.event', 'recovery.update')
->label('audits.resource', 'user/{response.userId}')
->label('audits.userId', '{response.userId}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'updateRecovery')
@ -2232,7 +2204,6 @@ App::post('/v1/account/verification')
->label('event', 'users.[userId].verification.[tokenId].create')
->label('audits.event', 'verification.create')
->label('audits.resource', 'user/{response.userId}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'createVerification')
@ -2343,7 +2314,6 @@ App::put('/v1/account/verification')
->label('event', 'users.[userId].verification.[tokenId].update')
->label('audits.event', 'verification.update')
->label('audits.resource', 'user/{response.userId}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'updateVerification')
@ -2402,7 +2372,6 @@ App::post('/v1/account/verification/phone')
->label('event', 'users.[userId].verification.[tokenId].create')
->label('audits.event', 'verification.create')
->label('audits.resource', 'user/{response.userId}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'createPhoneVerification')
@ -2487,7 +2456,6 @@ App::put('/v1/account/verification/phone')
->label('event', 'users.[userId].verification.[tokenId].update')
->label('audits.event', 'verification.update')
->label('audits.resource', 'user/{response.userId}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'updatePhoneVerification')

472
app/controllers/api/databases.php
View file

@ -18,7 +18,6 @@ use Utopia\Validator\ArrayList;
use Utopia\Validator\JSON;
use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\Database\DateTime;
use Utopia\Database\Query;
use Utopia\Database\Validator\Authorization;
use Utopia\Database\Validator\Key;
@ -153,7 +152,6 @@ App::post('/v1/databases')
->label('scope', 'databases.write')
->label('audits.event', 'database.create')
->label('audits.resource', 'database/{response.$id}')
->label('usage.metric', 'databases.{scope}.requests.create')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'create')
@ -225,7 +223,6 @@ App::get('/v1/databases')
->desc('List Databases')
->groups(['api', 'database'])
->label('scope', 'databases.read')
->label('usage.metric', 'databases.{scope}.requests.read')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'list')
@ -272,7 +269,6 @@ App::get('/v1/databases/:databaseId')
->desc('Get Database')
->groups(['api', 'database'])
->label('scope', 'databases.read')
->label('usage.metric', 'databases.{scope}.requests.read')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'get')
@ -387,7 +383,6 @@ App::put('/v1/databases/:databaseId')
->label('event', 'databases.[databaseId].update')
->label('audits.event', 'database.update')
->label('audits.resource', 'database/{response.$id}')
->label('usage.metric', 'databases.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'update')
@ -430,7 +425,6 @@ App::delete('/v1/databases/:databaseId')
->label('event', 'databases.[databaseId].delete')
->label('audits.event', 'database.delete')
->label('audits.resource', 'database/{request.databaseId}')
->label('usage.metric', 'databases.{scope}.requests.delete')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'delete')
@ -477,8 +471,6 @@ App::post('/v1/databases/:databaseId/collections')
->label('scope', 'collections.write')
->label('audits.event', 'collection.create')
->label('audits.resource', 'database/{request.databaseId}/collection/{response.$id}')
->label('usage.metric', 'collections.{scope}.requests.create')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'createCollection')
@ -542,8 +534,6 @@ App::get('/v1/databases/:databaseId/collections')
->desc('List Collections')
->groups(['api', 'database'])
->label('scope', 'collections.read')
->label('usage.metric', 'collections.{scope}.requests.read')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'listCollections')
@ -598,8 +588,6 @@ App::get('/v1/databases/:databaseId/collections/:collectionId')
->desc('Get Collection')
->groups(['api', 'database'])
->label('scope', 'collections.read')
->label('usage.metric', 'collections.{scope}.requests.read')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'getCollection')
@ -633,8 +621,6 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/logs')
->desc('List Collection Logs')
->groups(['api', 'database'])
->label('scope', 'collections.read')
->label('usage.metric', 'collections.{scope}.requests.read')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'listCollectionLogs')
@ -732,8 +718,6 @@ App::put('/v1/databases/:databaseId/collections/:collectionId')
->label('event', 'databases.[databaseId].collections.[collectionId].update')
->label('audits.event', 'collection.update')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
->label('usage.metric', 'collections.{scope}.requests.update')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'updateCollection')
@ -800,8 +784,6 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId')
->label('event', 'databases.[databaseId].collections.[collectionId].delete')
->label('audits.event', 'collection.delete')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
->label('usage.metric', 'collections.{scope}.requests.delete')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'deleteCollection')
@ -857,8 +839,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/string
->label('scope', 'collections.write')
->label('audits.event', 'attribute.create')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
->label('usage.metric', 'collections.{scope}.requests.update')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'createStringAttribute')
@ -907,8 +887,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/email'
->label('scope', 'collections.write')
->label('audits.event', 'attribute.create')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
->label('usage.metric', 'collections.{scope}.requests.update')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.namespace', 'databases')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.method', 'createEmailAttribute')
@ -951,8 +929,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/enum')
->label('scope', 'collections.write')
->label('audits.event', 'attribute.create')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
->label('usage.metric', 'collections.{scope}.requests.update')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.namespace', 'databases')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.method', 'createEnumAttribute')
@ -1011,8 +987,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/ip')
->label('scope', 'collections.write')
->label('audits.event', 'attribute.create')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
->label('usage.metric', 'collections.{scope}.requests.update')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.namespace', 'databases')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.method', 'createIpAttribute')
@ -1055,8 +1029,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/url')
->label('scope', 'collections.write')
->label('audits.event', 'attribute.create')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
->label('usage.metric', 'collections.{scope}.requests.update')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.namespace', 'databases')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.method', 'createUrlAttribute')
@ -1099,8 +1071,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/intege
->label('scope', 'collections.write')
->label('audits.event', 'attribute.create')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
->label('usage.metric', 'collections.{scope}.requests.update')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.namespace', 'databases')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.method', 'createIntegerAttribute')
@ -1172,8 +1142,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/float'
->label('scope', 'collections.write')
->label('audits.event', 'attribute.create')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
->label('usage.metric', 'collections.{scope}.requests.update')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.namespace', 'databases')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.method', 'createFloatAttribute')
@ -1248,8 +1216,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/boolea
->label('scope', 'collections.write')
->label('audits.event', 'attribute.create')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
->label('usage.metric', 'collections.{scope}.requests.update')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.namespace', 'databases')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.method', 'createBooleanAttribute')
@ -1292,8 +1258,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/dateti
->label('scope', 'collections.write')
->label('audits.event', 'attribute.create')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
->label('usage.metric', 'collections.{scope}.requests.update')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.namespace', 'databases')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.method', 'createDatetimeAttribute')
@ -1334,8 +1298,6 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/attributes')
->desc('List Attributes')
->groups(['api', 'database'])
->label('scope', 'collections.read')
->label('usage.metric', 'collections.{scope}.requests.read')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'listAttributes')
@ -1373,8 +1335,6 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/attributes/:key')
->desc('Get Attribute')
->groups(['api', 'database'])
->label('scope', 'collections.read')
->label('usage.metric', 'collections.{scope}.requests.read')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'getAttribute')
@ -1447,8 +1407,6 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/attributes/:key
->label('event', 'databases.[databaseId].collections.[collectionId].attributes.[attributeId].delete')
->label('audits.event', 'attribute.delete')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
->label('usage.metric', 'collections.{scope}.requests.update')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'deleteAttribute')
@ -1535,8 +1493,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/indexes')
->label('scope', 'collections.write')
->label('audits.event', 'index.create')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
->label('usage.metric', 'collections.{scope}.requests.update')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'createIndex')
@ -1682,8 +1638,6 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/indexes')
->desc('List Indexes')
->groups(['api', 'database'])
->label('scope', 'collections.read')
->label('usage.metric', 'collections.{scope}.requests.read')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'listIndexes')
@ -1721,8 +1675,6 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/indexes/:key')
->desc('Get Index')
->groups(['api', 'database'])
->label('scope', 'collections.read')
->label('usage.metric', 'collections.{scope}.requests.read')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'getIndex')
@ -1771,8 +1723,6 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/indexes/:key')
->label('event', 'databases.[databaseId].collections.[collectionId].indexes.[indexId].delete')
->label('audits.event', 'index.delete')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
->label('usage.metric', 'collections.{scope}.requests.update')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'deleteIndex')
@ -1839,8 +1789,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents')
->label('scope', 'documents.write')
->label('audits.event', 'document.create')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
->label('usage.metric', 'documents.{scope}.requests.create')
->label('usage.params', ['databaseId:{request.databaseId}', 'collectionId:{request.collectionId}'])
->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}')
->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT * 2)
->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT)
@ -1964,8 +1912,6 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents')
->desc('List Documents')
->groups(['api', 'database'])
->label('scope', 'documents.read')
->label('usage.metric', 'documents.{scope}.requests.read')
->label('usage.params', ['databaseId:{request.databaseId}', 'collectionId:{request.collectionId}'])
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'listDocuments')
@ -2061,8 +2007,6 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen
->desc('Get Document')
->groups(['api', 'database'])
->label('scope', 'documents.read')
->label('usage.metric', 'documents.{scope}.requests.read')
->label('usage.params', ['databaseId:{request.databaseId}', 'collectionId:{request.collectionId}'])
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'getDocument')
@ -2123,8 +2067,6 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen
->desc('List Document Logs')
->groups(['api', 'database'])
->label('scope', 'documents.read')
->label('usage.metric', 'documents.{scope}.requests.read')
->label('usage.params', ['databaseId:{request.databaseId}', 'collectionId:{request.collectionId}'])
->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'listDocumentLogs')
@ -2227,8 +2169,6 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum
->label('scope', 'documents.write')
->label('audits.event', 'document.update')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}/document/{response.$id}')
->label('usage.metric', 'documents.{scope}.requests.update')
->label('usage.params', ['databaseId:{request.databaseId}', 'collectionId:{request.collectionId}'])
->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}')
->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT * 2)
->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT)
@ -2361,8 +2301,6 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu
->label('event', 'databases.[databaseId].collections.[collectionId].documents.[documentId].delete')
->label('audits.event', 'document.delete')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}/document/{request.documentId}')
->label('usage.metric', 'documents.{scope}.requests.delete')
->label('usage.params', ['databaseId:{request.databaseId}', 'collectionId:{request.collectionId}'])
->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}')
->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT)
->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT)
@ -2460,107 +2398,57 @@ App::get('/v1/databases/usage')
->inject('dbForProject')
->action(function (string $range, Response $response, Database $dbForProject) {
$usage = [];
if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') {
$periods = [
'24h' => [
'period' => '1h',
'limit' => 24,
],
'7d' => [
'period' => '1d',
'limit' => 7,
],
'30d' => [
'period' => '1d',
'limit' => 30,
],
'90d' => [
'period' => '1d',
'limit' => 90,
],
];
$periods = Config::getParam('usage', []);
$stats = $usage = [];
$days = $periods[$range];
$metrics = [
METRIC_DATABASES,
METRIC_COLLECTIONS,
METRIC_DOCUMENTS,
];
$metrics = [
'databases.$all.count.total',
'documents.$all.count.total',
'collections.$all.count.total',
'databases.$all.requests.create',
'databases.$all.requests.read',
'databases.$all.requests.update',
'databases.$all.requests.delete',
'collections.$all.requests.create',
'collections.$all.requests.read',
'collections.$all.requests.update',
'collections.$all.requests.delete',
'documents.$all.requests.create',
'documents.$all.requests.read',
'documents.$all.requests.update',
'documents.$all.requests.delete'
];
$stats = [];
Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) {
foreach ($metrics as $metric) {
$limit = $periods[$range]['limit'];
$period = $periods[$range]['period'];
$requestDocs = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric] = [];
foreach ($requestDocs as $requestDoc) {
$stats[$metric][] = [
'value' => $requestDoc->getAttribute('value'),
'date' => $requestDoc->getAttribute('time'),
];
}
// backfill metrics with empty values for graphs
$backfill = $limit - \count($requestDocs);
while ($backfill > 0) {
$last = $limit - $backfill - 1; // array index of last added metric
$diff = match ($period) { // convert period to seconds for unix timestamp math
'1h' => 3600,
'1d' => 86400,
};
$stats[$metric][] = [
'value' => 0,
'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)),
];
$backfill--;
}
// Added 3'rd level to Index [period, metric, time] because of order by.
$stats[$metric] = array_reverse($stats[$metric]);
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
foreach ($metrics as $metric) {
$limit = $days['limit'];
$period = $days['period'];
$results = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric] = [];
foreach ($results as $result) {
$stats[$metric][$result->getAttribute('time')] = [
'value' => $result->getAttribute('value'),
];
}
});
}
});
$usage = new Document([
'range' => $range,
'databasesCount' => $stats['databases.$all.count.total'] ?? [],
'documentsCount' => $stats['documents.$all.count.total'] ?? [],
'collectionsCount' => $stats['collections.$all.count.total'] ?? [],
'documentsCreate' => $stats['documents.$all.requests.create'] ?? [],
'documentsRead' => $stats['documents.$all.requests.read'] ?? [],
'documentsUpdate' => $stats['documents.$all.requests.update'] ?? [],
'documentsDelete' => $stats['documents.$all.requests.delete'] ?? [],
'collectionsCreate' => $stats['collections.$all.requests.create'] ?? [],
'collectionsRead' => $stats['collections.$all.requests.read'] ?? [],
'collectionsUpdate' => $stats['collections.$all.requests.update'] ?? [],
'collectionsDelete' => $stats['collections.$all.requests.delete'] ?? [],
'databasesCreate' => $stats['databases.$all.requests.create'] ?? [],
'databasesRead' => $stats['databases.$all.requests.read'] ?? [],
'databasesUpdate' => $stats['databases.$all.requests.update'] ?? [],
'databasesDelete' => $stats['databases.$all.requests.delete'] ?? [],
]);
$format = match ($days['period']) {
'1h' => 'Y-m-d\TH:00:00.000P',
'1d' => 'Y-m-d\T00:00:00.000P',
};
foreach ($metrics as $metric) {
$usage[$metric] = [];
$leap = time() - ($days['limit'] * $days['factor']);
while ($leap < time()) {
$leap += $days['factor'];
$formatDate = date($format, $leap);
$usage[$metric][] = [
'value' => $stats[$metric][$formatDate]['value'] ?? 0,
'date' => $formatDate,
];
}
$response->dynamic($usage, Response::MODEL_USAGE_DATABASES);
}
$response->dynamic(new Document([
'range' => $range,
'databasesTotal' => $usage[$metrics[0]],
'collectionsTotal' => $usage[$metrics[1]],
'documentsTotal' => $usage[$metrics[2]],
]), Response::MODEL_USAGE_DATABASES);
});
App::get('/v1/databases/:databaseId/usage')
@ -2579,97 +2467,62 @@ App::get('/v1/databases/:databaseId/usage')
->inject('dbForProject')
->action(function (string $databaseId, string $range, Response $response, Database $dbForProject) {
$usage = [];
if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') {
$periods = [
'24h' => [
'period' => '1h',
'limit' => 24,
],
'7d' => [
'period' => '1d',
'limit' => 7,
],
'30d' => [
'period' => '1d',
'limit' => 30,
],
'90d' => [
'period' => '1d',
'limit' => 90,
],
];
$database = $dbForProject->getDocument('databases', $databaseId);
$metrics = [
'collections.' . $databaseId . '.count.total',
'collections.' . $databaseId . '.requests.create',
'collections.' . $databaseId . '.requests.read',
'collections.' . $databaseId . '.requests.update',
'collections.' . $databaseId . '.requests.delete',
'documents.' . $databaseId . '.count.total',
'documents.' . $databaseId . '.requests.create',
'documents.' . $databaseId . '.requests.read',
'documents.' . $databaseId . '.requests.update',
'documents.' . $databaseId . '.requests.delete'
];
$stats = [];
Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) {
foreach ($metrics as $metric) {
$limit = $periods[$range]['limit'];
$period = $periods[$range]['period'];
$requestDocs = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric] = [];
foreach ($requestDocs as $requestDoc) {
$stats[$metric][] = [
'value' => $requestDoc->getAttribute('value'),
'date' => $requestDoc->getAttribute('time'),
];
}
// backfill metrics with empty values for graphs
$backfill = $limit - \count($requestDocs);
while ($backfill > 0) {
$last = $limit - $backfill - 1; // array index of last added metric
$diff = match ($period) { // convert period to seconds for unix timestamp math
'1h' => 3600,
'1d' => 86400,
};
$stats[$metric][] = [
'value' => 0,
'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)),
];
$backfill--;
}
// TODO@kodumbeats explore performance if query is ordered by time ASC
$stats[$metric] = array_reverse($stats[$metric]);
}
});
$usage = new Document([
'range' => $range,
'collectionsCount' => $stats["collections.{$databaseId}.count.total"] ?? [],
'collectionsCreate' => $stats["collections.{$databaseId}.requests.create"] ?? [],
'collectionsRead' => $stats["collections.{$databaseId}.requests.read"] ?? [],
'collectionsUpdate' => $stats["collections.{$databaseId}.requests.update"] ?? [],
'collectionsDelete' => $stats["collections.{$databaseId}.requests.delete"] ?? [],
'documentsCount' => $stats["documents.{$databaseId}.count.total"] ?? [],
'documentsCreate' => $stats["documents.{$databaseId}.requests.create"] ?? [],
'documentsRead' => $stats["documents.{$databaseId}.requests.read"] ?? [],
'documentsUpdate' => $stats["documents.{$databaseId}.requests.update"] ?? [],
'documentsDelete' => $stats["documents.{$databaseId}.requests.delete"] ?? [],
]);
if ($database->isEmpty()) {
throw new Exception(Exception::DATABASE_NOT_FOUND);
}
$response->dynamic($usage, Response::MODEL_USAGE_DATABASE);
$periods = Config::getParam('usage', []);
$stats = $usage = [];
$days = $periods[$range];
$metrics = [
str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_COLLECTIONS),
str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_DOCUMENTS),
];
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
foreach ($metrics as $metric) {
$limit = $days['limit'];
$period = $days['period'];
$results = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric] = [];
foreach ($results as $result) {
$stats[$metric][$result->getAttribute('time')] = [
'value' => $result->getAttribute('value'),
];
}
}
});
$format = match ($days['period']) {
'1h' => 'Y-m-d\TH:00:00.000P',
'1d' => 'Y-m-d\T00:00:00.000P',
};
foreach ($metrics as $metric) {
$usage[$metric] = [];
$leap = time() - ($days['limit'] * $days['factor']);
while ($leap < time()) {
$leap += $days['factor'];
$formatDate = date($format, $leap);
$usage[$metric][] = [
'value' => $stats[$metric][$formatDate]['value'] ?? 0,
'date' => $formatDate,
];
}
}
$response->dynamic(new Document([
'range' => $range,
'collectionsTotal' => $usage[$metrics[0]],
'documentsTotal' => $usage[$metrics[1]],
]), Response::MODEL_USAGE_DATABASE);
});
App::get('/v1/databases/:databaseId/collections/:collectionId/usage')
@ -2690,93 +2543,60 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/usage')
->inject('dbForProject')
->action(function (string $databaseId, string $range, string $collectionId, Response $response, Database $dbForProject) {
$database = $dbForProject->getDocument('databases', $databaseId);
$collectionDocument = $dbForProject->getDocument('database_' . $database->getInternalId(), $collectionId);
$collection = $dbForProject->getCollection('database_' . $database->getInternalId() . '_collection_' . $collectionDocument->getInternalId());
$database = $dbForProject->getDocument('databases', $databaseId);
$collectionDocument = $dbForProject->getDocument('database_' . $database->getInternalId(), $collectionId);
$collection = $dbForProject->getCollection('database_' . $database->getInternalId() . '_collection_' . $collectionDocument->getInternalId());
if ($collection->isEmpty()) {
throw new Exception(Exception::COLLECTION_NOT_FOUND);
}
$usage = [];
if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') {
$periods = [
'24h' => [
'period' => '1h',
'limit' => 24,
],
'7d' => [
'period' => '1d',
'limit' => 7,
],
'30d' => [
'period' => '1d',
'limit' => 30,
],
'90d' => [
'period' => '1d',
'limit' => 90,
],
];
$periods = Config::getParam('usage', []);
$stats = $usage = [];
$days = $periods[$range];
$metrics = [
str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collectionDocument->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS),
];
$metrics = [
"documents.{$databaseId}/{$collectionId}.count.total",
"documents.{$databaseId}/{$collectionId}.requests.create",
"documents.{$databaseId}/{$collectionId}.requests.read",
"documents.{$databaseId}/{$collectionId}.requests.update",
"documents.{$databaseId}/{$collectionId}.requests.delete",
];
$stats = [];
Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) {
foreach ($metrics as $metric) {
$limit = $periods[$range]['limit'];
$period = $periods[$range]['period'];
$requestDocs = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric] = [];
foreach ($requestDocs as $requestDoc) {
$stats[$metric][] = [
'value' => $requestDoc->getAttribute('value'),
'date' => $requestDoc->getAttribute('time'),
];
}
// backfill metrics with empty values for graphs
$backfill = $limit - \count($requestDocs);
while ($backfill > 0) {
$last = $limit - $backfill - 1; // array index of last added metric
$diff = match ($period) { // convert period to seconds for unix timestamp math
'1h' => 3600,
'1d' => 86400,
};
$stats[$metric][] = [
'value' => 0,
'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)),
];
$backfill--;
}
$stats[$metric] = array_reverse($stats[$metric]);
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
foreach ($metrics as $metric) {
$limit = $days['limit'];
$period = $days['period'];
$results = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric] = [];
foreach ($results as $result) {
$stats[$metric][$result->getAttribute('time')] = [
'value' => $result->getAttribute('value'),
];
}
});
}
});
$usage = new Document([
'range' => $range,
'documentsCount' => $stats["documents.{$databaseId}/{$collectionId}.count.total"] ?? [],
'documentsCreate' => $stats["documents.{$databaseId}/{$collectionId}.requests.create"] ?? [],
'documentsRead' => $stats["documents.{$databaseId}/{$collectionId}.requests.read"] ?? [],
'documentsUpdate' => $stats["documents.{$databaseId}/{$collectionId}.requests.update"] ?? [],
'documentsDelete' => $stats["documents.{$databaseId}/{$collectionId}.requests.delete" ?? []]
]);
$format = match ($days['period']) {
'1h' => 'Y-m-d\TH:00:00.000P',
'1d' => 'Y-m-d\T00:00:00.000P',
};
foreach ($metrics as $metric) {
$usage[$metric] = [];
$leap = time() - ($days['limit'] * $days['factor']);
while ($leap < time()) {
$leap += $days['factor'];
$formatDate = date($format, $leap);
$usage[$metric][] = [
'value' => $stats[$metric][$formatDate]['value'] ?? 0,
'date' => $formatDate,
];
}
}
$response->dynamic($usage, Response::MODEL_USAGE_COLLECTION);
$response->dynamic(new Document([
'range' => $range,
'documentsTotal' => $usage[$metrics[0]],
]), Response::MODEL_USAGE_COLLECTION);
});

300
app/controllers/api/functions.php
View file

@ -6,6 +6,7 @@ use Appwrite\Event\Build;
use Appwrite\Event\Delete;
use Appwrite\Event\Event;
use Appwrite\Event\Func;
use Appwrite\Event\Usage;
use Appwrite\Event\Validator\Event as ValidatorEvent;
use Appwrite\Extend\Exception;
use Appwrite\Utopia\Database\Validator\CustomId;
@ -13,7 +14,6 @@ use Utopia\Database\Helpers\ID;
use Utopia\Database\Helpers\Permission;
use Utopia\Database\Helpers\Role;
use Utopia\Database\Validator\UID;
use Appwrite\Usage\Stats;
use Utopia\Storage\Device;
use Utopia\Storage\Validator\File;
use Utopia\Storage\Validator\FileExt;
@ -85,6 +85,7 @@ App::post('/v1/functions')
'deployment' => '',
'events' => $events,
'schedule' => $schedule,
'scheduleInternalId' => '',
'scheduleUpdatedAt' => DateTime::now(),
'timeout' => $timeout,
'search' => implode(' ', [$functionId, $name, $runtime])
@ -95,6 +96,7 @@ App::post('/v1/functions')
'region' => App::getEnv('_APP_REGION', 'default'), // Todo replace with projects region
'resourceType' => 'function',
'resourceId' => $function->getId(),
'resourceInternalId' => $function->getInternalId(),
'resourceUpdatedAt' => DateTime::now(),
'projectId' => $project->getId(),
'schedule' => $function->getAttribute('schedule'),
@ -232,92 +234,66 @@ App::get('/v1/functions/:functionId/usage')
throw new Exception(Exception::FUNCTION_NOT_FOUND);
}
$usage = [];
if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') {
$periods = [
'24h' => [
'period' => '1h',
'limit' => 24,
],
'7d' => [
'period' => '1d',
'limit' => 7,
],
'30d' => [
'period' => '1d',
'limit' => 30,
],
'90d' => [
'period' => '1d',
'limit' => 90,
],
];
$periods = Config::getParam('usage', []);
$stats = $usage = [];
$days = $periods[$range];
$metrics = [
str_replace(['{resourceType}', '{resourceInternalId}'], ['function', $function->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS),
str_replace(['{resourceType}', '{resourceInternalId}'], ['function', $function->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS_STORAGE),
str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS),
str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_STORAGE),
str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE),
str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS),
str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE),
];
$metrics = [
"executions.$functionId.compute.total",
"executions.$functionId.compute.success",
"executions.$functionId.compute.failure",
"executions.$functionId.compute.time",
"builds.$functionId.compute.total",
"builds.$functionId.compute.success",
"builds.$functionId.compute.failure",
"builds.$functionId.compute.time",
];
$stats = [];
Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) {
foreach ($metrics as $metric) {
$limit = $periods[$range]['limit'];
$period = $periods[$range]['period'];
$requestDocs = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric] = [];
foreach ($requestDocs as $requestDoc) {
$stats[$metric][] = [
'value' => $requestDoc->getAttribute('value'),
'date' => $requestDoc->getAttribute('time'),
];
}
// backfill metrics with empty values for graphs
$backfill = $limit - \count($requestDocs);
while ($backfill > 0) {
$last = $limit - $backfill - 1; // array index of last added metric
$diff = match ($period) { // convert period to seconds for unix timestamp math
'1h' => 3600,
'1d' => 86400,
};
$stats[$metric][] = [
'value' => 0,
'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)),
];
$backfill--;
}
$stats[$metric] = array_reverse($stats[$metric]);
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
foreach ($metrics as $metric) {
$limit = $days['limit'];
$period = $days['period'];
$results = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric] = [];
foreach ($results as $result) {
$stats[$metric][$result->getAttribute('time')] = [
'value' => $result->getAttribute('value'),
];
}
});
}
});
$usage = new Document([
'range' => $range,
'executionsTotal' => $stats["executions.$functionId.compute.total"] ?? [],
'executionsFailure' => $stats["executions.$functionId.compute.failure"] ?? [],
'executionsSuccesse' => $stats["executions.$functionId.compute.success"] ?? [],
'executionsTime' => $stats["executions.$functionId.compute.time"] ?? [],
'buildsTotal' => $stats["builds.$functionId.compute.total"] ?? [],
'buildsFailure' => $stats["builds.$functionId.compute.failure"] ?? [],
'buildsSuccess' => $stats["builds.$functionId.compute.success"] ?? [],
'buildsTime' => $stats["builds.$functionId.compute.time" ?? []]
]);
$format = match ($days['period']) {
'1h' => 'Y-m-d\TH:00:00.000P',
'1d' => 'Y-m-d\T00:00:00.000P',
};
foreach ($metrics as $metric) {
$usage[$metric] = [];
$leap = time() - ($days['limit'] * $days['factor']);
while ($leap < time()) {
$leap += $days['factor'];
$formatDate = date($format, $leap);
$usage[$metric][] = [
'value' => $stats[$metric][$formatDate]['value'] ?? 0,
'date' => $formatDate,
];
}
}
$response->dynamic($usage, Response::MODEL_USAGE_FUNCTION);
$response->dynamic(new Document([
'range' => $range,
'deploymentsTotal' => $usage[$metrics[0]],
'deploymentsStorage' => $usage[$metrics[1]],
'buildsTotal' => $usage[$metrics[2]],
'buildsStorage' => $usage[$metrics[3]],
'buildsTime' => $usage[$metrics[4]],
'executionsTotal' => $usage[$metrics[5]],
'executionsTime' => $usage[$metrics[6]],
]), Response::MODEL_USAGE_FUNCTION);
});
App::get('/v1/functions/usage')
@ -335,92 +311,67 @@ App::get('/v1/functions/usage')
->inject('dbForProject')
->action(function (string $range, Response $response, Database $dbForProject) {
$usage = [];
if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') {
$periods = [
'24h' => [
'period' => '1h',
'limit' => 24,
],
'7d' => [
'period' => '1d',
'limit' => 7,
],
'30d' => [
'period' => '1d',
'limit' => 30,
],
'90d' => [
'period' => '1d',
'limit' => 90,
],
];
$periods = Config::getParam('usage', []);
$stats = $usage = [];
$days = $periods[$range];
$metrics = [
METRIC_FUNCTIONS,
METRIC_DEPLOYMENTS,
METRIC_DEPLOYMENTS_STORAGE,
METRIC_BUILDS,
METRIC_BUILDS_STORAGE,
METRIC_BUILDS_COMPUTE,
METRIC_EXECUTIONS,
METRIC_EXECUTIONS_COMPUTE,
];
$metrics = [
'executions.$all.compute.total',
'executions.$all.compute.failure',
'executions.$all.compute.success',
'executions.$all.compute.time',
'builds.$all.compute.total',
'builds.$all.compute.failure',
'builds.$all.compute.success',
'builds.$all.compute.time',
];
$stats = [];
Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) {
foreach ($metrics as $metric) {
$limit = $periods[$range]['limit'];
$period = $periods[$range]['period'];
$requestDocs = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric] = [];
foreach ($requestDocs as $requestDoc) {
$stats[$metric][] = [
'value' => $requestDoc->getAttribute('value'),
'date' => $requestDoc->getAttribute('time'),
];
}
// backfill metrics with empty values for graphs
$backfill = $limit - \count($requestDocs);
while ($backfill > 0) {
$last = $limit - $backfill - 1; // array index of last added metric
$diff = match ($period) { // convert period to seconds for unix timestamp math
'1h' => 3600,
'1d' => 86400,
};
$stats[$metric][] = [
'value' => 0,
'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)),
];
$backfill--;
}
$stats[$metric] = array_reverse($stats[$metric]);
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
foreach ($metrics as $metric) {
$limit = $days['limit'];
$period = $days['period'];
$results = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric] = [];
foreach ($results as $result) {
$stats[$metric][$result->getAttribute('time')] = [
'value' => $result->getAttribute('value'),
];
}
});
}
});
$usage = new Document([
'range' => $range,
'executionsTotal' => $stats[$metrics[0]] ?? [],
'executionsFailure' => $stats[$metrics[1]] ?? [],
'executionsSuccess' => $stats[$metrics[2]] ?? [],
'executionsTime' => $stats[$metrics[3]] ?? [],
'buildsTotal' => $stats[$metrics[4]] ?? [],
'buildsFailure' => $stats[$metrics[5]] ?? [],
'buildsSuccess' => $stats[$metrics[6]] ?? [],
'buildsTime' => $stats[$metrics[7]] ?? [],
]);
$format = match ($days['period']) {
'1h' => 'Y-m-d\TH:00:00.000P',
'1d' => 'Y-m-d\T00:00:00.000P',
};
foreach ($metrics as $metric) {
$usage[$metric] = [];
$leap = time() - ($days['limit'] * $days['factor']);
while ($leap < time()) {
$leap += $days['factor'];
$formatDate = date($format, $leap);
$usage[$metric][] = [
'value' => $stats[$metric][$formatDate]['value'] ?? 0,
'date' => $formatDate,
];
}
$response->dynamic($usage, Response::MODEL_USAGE_FUNCTIONS);
}
$response->dynamic(new Document([
'range' => $range,
'functionsTotal' => $usage[$metrics[0]],
'deploymentsTotal' => $usage[$metrics[1]],
'deploymentsStorage' => $usage[$metrics[2]],
'buildsTotal' => $usage[$metrics[3]],
'buildsStorage' => $usage[$metrics[4]],
'buildsTime' => $usage[$metrics[5]],
'executionsTotal' => $usage[$metrics[6]],
'executionsTime' => $usage[$metrics[7]],
]), Response::MODEL_USAGE_FUNCTIONS);
});
App::put('/v1/functions/:functionId')
@ -704,7 +655,6 @@ App::post('/v1/functions/:functionId/deployments')
// Save to storage
$fileSize ??= $deviceLocal->getFileSize($fileTmpName);
$path = $deviceFunctions->getPath($deploymentId . '.' . \pathinfo($fileName, PATHINFO_EXTENSION));
$deployment = $dbForProject->getDocument('deployments', $deploymentId);
$metadata = ['content_type' => $deviceLocal->getFileMimeType($fileTmpName)];
@ -752,6 +702,7 @@ App::post('/v1/functions/:functionId/deployments')
'resourceInternalId' => $function->getInternalId(),
'resourceId' => $function->getId(),
'resourceType' => 'functions',
'buildInternalId' => '',
'entrypoint' => $entrypoint,
'path' => $path,
'size' => $fileSize,
@ -783,6 +734,7 @@ App::post('/v1/functions/:functionId/deployments')
'resourceInternalId' => $function->getInternalId(),
'resourceId' => $function->getId(),
'resourceType' => 'functions',
'buildInternalId' => '',
'entrypoint' => $entrypoint,
'path' => $path,
'size' => $fileSize,
@ -1075,10 +1027,10 @@ App::post('/v1/functions/:functionId/executions')
->inject('dbForProject')
->inject('user')
->inject('events')
->inject('usage')
->inject('mode')
->inject('queueForFunctions')
->action(function (string $functionId, string $data, bool $async, Response $response, Document $project, Database $dbForProject, Document $user, Event $events, Stats $usage, string $mode, Func $queueForFunctions) {
->inject('queueForUsage')
->action(function (string $functionId, string $data, bool $async, Response $response, Document $project, Database $dbForProject, Document $user, Event $events, string $mode, Func $queueForFunctions, Usage $queueForUsage) {
$function = Authorization::skip(fn () => $dbForProject->getDocument('functions', $functionId));
@ -1222,6 +1174,13 @@ App::post('/v1/functions/:functionId/executions')
$execution->setAttribute('stdout', $executionResponse['stdout']);
$execution->setAttribute('stderr', $executionResponse['stderr']);
$execution->setAttribute('duration', $executionResponse['duration']);
/**
* Sync execution compute usage from
*/
$queueForUsage
->addMetric(METRIC_EXECUTIONS_COMPUTE, (int)($executionResponse['duration'] * 1000))// per project
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE), (int)($executionResponse['duration'] * 1000))// per function
;
} catch (\Throwable $th) {
$interval = (new \DateTime())->diff(new \DateTime($execution->getCreatedAt()));
$execution
@ -1234,13 +1193,6 @@ App::post('/v1/functions/:functionId/executions')
Authorization::skip(fn () => $dbForProject->updateDocument('executions', $executionId, $execution));
// TODO revise this later using route label
$usage
->setParam('functionId', $function->getId())
->setParam('executions.{scope}.compute', 1)
->setParam('executionStatus', $execution->getAttribute('status', ''))
->setParam('executionTime', $execution->getAttribute('duration')); // ms
$roles = Authorization::getRoles();
$isPrivilegedUser = Auth::isPrivilegedUser($roles);
$isAppUser = Auth::isAppUser($roles);

140
app/controllers/api/project.php
View file

@ -2,6 +2,7 @@
use Appwrite\Utopia\Response;
use Utopia\App;
use Utopia\Config\Config;
use Utopia\Database\Database;
use Utopia\Database\DateTime;
use Utopia\Database\Document;
@ -23,90 +24,71 @@ App::get('/v1/project/usage')
->inject('response')
->inject('dbForProject')
->action(function (string $range, Response $response, Database $dbForProject) {
$usage = [];
if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') {
$periods = [
'24h' => [
'period' => '1h',
'limit' => 24,
],
'7d' => [
'period' => '1d',
'limit' => 7,
],
'30d' => [
'period' => '1d',
'limit' => 30,
],
'90d' => [
'period' => '1d',
'limit' => 90,
],
];
$metrics = [
'project.$all.network.requests',
'project.$all.network.bandwidth',
'project.$all.storage.size',
'users.$all.count.total',
'databases.$all.count.total',
'documents.$all.count.total',
'executions.$all.compute.total',
'buckets.$all.count.total'
];
$periods = Config::getParam('usage', []);
$stats = $usage = [];
$days = $periods[$range];
$metrics = [
METRIC_NETWORK_REQUESTS,
METRIC_NETWORK_INBOUND,
METRIC_NETWORK_OUTBOUND,
METRIC_EXECUTIONS,
METRIC_DOCUMENTS,
METRIC_DATABASES,
METRIC_USERS,
METRIC_BUCKETS,
METRIC_FILES_STORAGE
];
$stats = [];
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
foreach ($metrics as $metric) {
$limit = $days['limit'];
$period = $days['period'];
$results = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) {
foreach ($metrics as $metric) {
$limit = $periods[$range]['limit'];
$period = $periods[$range]['period'];
$requestDocs = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric] = [];
foreach ($requestDocs as $requestDoc) {
$stats[$metric][] = [
'value' => $requestDoc->getAttribute('value'),
'date' => $requestDoc->getAttribute('time'),
];
}
// backfill metrics with empty values for graphs
$backfill = $limit - \count($requestDocs);
while ($backfill > 0) {
$last = $limit - $backfill - 1; // array index of last added metric
$diff = match ($period) { // convert period to seconds for unix timestamp math
'1h' => 3600,
'1d' => 86400,
};
$stats[$metric][] = [
'value' => 0,
'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)),
];
$backfill--;
}
$stats[$metric] = array_reverse($stats[$metric]);
$stats[$metric] = [];
foreach ($results as $result) {
$stats[$metric][$result->getAttribute('time')] = [
'value' => $result->getAttribute('value'),
];
}
});
}
});
$usage = new Document([
'range' => $range,
'requests' => $stats[$metrics[0]] ?? [],
'network' => $stats[$metrics[1]] ?? [],
'storage' => $stats[$metrics[2]] ?? [],
'users' => $stats[$metrics[3]] ?? [],
'databases' => $stats[$metrics[4]] ?? [],
'documents' => $stats[$metrics[5]] ?? [],
'executions' => $stats[$metrics[6]] ?? [],
'buckets' => $stats[$metrics[7]] ?? [],
]);
$format = match ($days['period']) {
'1h' => 'Y-m-d\TH:00:00.000P',
'1d' => 'Y-m-d\T00:00:00.000P',
};
foreach ($metrics as $metric) {
$usage[$metric] = [];
$leap = time() - ($days['limit'] * $days['factor']);
while ($leap < time()) {
$leap += $days['factor'];
$formatDate = date($format, $leap);
$usage[$metric][] = [
'value' => $stats[$metric][$formatDate]['value'] ?? 0,
'date' => $formatDate,
];
}
}
$response->dynamic($usage, Response::MODEL_USAGE_PROJECT);
$response->dynamic(new Document([
'range' => $range,
'requestsTotal' => ($usage[$metrics[0]]),
'network' => ($usage[$metrics[1]] + $usage[$metrics[2]]),
'executionsTotal' => $usage[$metrics[3]],
'documentsTotal' => $usage[$metrics[4]],
'databasesTotal' => $usage[$metrics[5]],
'usersTotal' => $usage[$metrics[6]],
'bucketsTotal' => $usage[$metrics[7]],
'filesStorage' => $usage[$metrics[8]],
]), Response::MODEL_USAGE_PROJECT);
});

325
app/controllers/api/storage.php
View file

@ -51,7 +51,6 @@ App::post('/v1/storage/buckets')
->label('event', 'buckets.[bucketId].create')
->label('audits.event', 'bucket.create')
->label('audits.resource', 'bucket/{response.$id}')
->label('usage.metric', 'buckets.{scope}.requests.create')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'storage')
->label('sdk.method', 'createBucket')
@ -147,7 +146,6 @@ App::get('/v1/storage/buckets')
->desc('List buckets')
->groups(['api', 'storage'])
->label('scope', 'buckets.read')
->label('usage.metric', 'buckets.{scope}.requests.read')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'storage')
->label('sdk.method', 'listBuckets')
@ -194,7 +192,6 @@ App::get('/v1/storage/buckets/:bucketId')
->desc('Get Bucket')
->groups(['api', 'storage'])
->label('scope', 'buckets.read')
->label('usage.metric', 'buckets.{scope}.requests.read')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'storage')
->label('sdk.method', 'getBucket')
@ -223,7 +220,6 @@ App::put('/v1/storage/buckets/:bucketId')
->label('event', 'buckets.[bucketId].update')
->label('audits.event', 'bucket.update')
->label('audits.resource', 'bucket/{response.$id}')
->label('usage.metric', 'buckets.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'storage')
->label('sdk.method', 'updateBucket')
@ -290,7 +286,6 @@ App::delete('/v1/storage/buckets/:bucketId')
->label('audits.event', 'bucket.delete')
->label('event', 'buckets.[bucketId].delete')
->label('audits.resource', 'bucket/{request.bucketId}')
->label('usage.metric', 'buckets.{scope}.requests.delete')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'storage')
->label('sdk.method', 'deleteBucket')
@ -333,8 +328,6 @@ App::post('/v1/storage/buckets/:bucketId/files')
->label('audits.event', 'file.create')
->label('event', 'buckets.[bucketId].files.[fileId].create')
->label('audits.resource', 'file/{response.$id}')
->label('usage.metric', 'files.{scope}.requests.create')
->label('usage.params', ['bucketId:{request.bucketId}'])
->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}')
->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT)
->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT)
@ -672,8 +665,6 @@ App::get('/v1/storage/buckets/:bucketId/files')
->groups(['api', 'storage'])
->label('scope', 'files.read')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
->label('usage.metric', 'files.{scope}.requests.read')
->label('usage.params', ['bucketId:{request.bucketId}'])
->label('sdk.namespace', 'storage')
->label('sdk.method', 'listFiles')
->label('sdk.description', '/docs/references/storage/list-files.md')
@ -749,8 +740,6 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId')
->groups(['api', 'storage'])
->label('scope', 'files.read')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
->label('usage.metric', 'files.{scope}.requests.read')
->label('usage.params', ['bucketId:{request.bucketId}'])
->label('sdk.namespace', 'storage')
->label('sdk.method', 'getFile')
->label('sdk.description', '/docs/references/storage/get-file.md')
@ -798,8 +787,6 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview')
->label('cache', true)
->label('cache.resourceType', 'bucket/{request.bucketId}')
->label('cache.resource', 'file/{request.fileId}')
->label('usage.metric', 'files.{scope}.requests.read')
->label('usage.params', ['bucketId:{request.bucketId}'])
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'storage')
->label('sdk.method', 'getFilePreview')
@ -961,8 +948,6 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/download')
->desc('Get File for Download')
->groups(['api', 'storage'])
->label('scope', 'files.read')
->label('usage.metric', 'files.{scope}.requests.read')
->label('usage.params', ['bucketId:{request.bucketId}'])
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'storage')
->label('sdk.method', 'getFileDownload')
@ -1101,8 +1086,6 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/view')
->desc('Get File for View')
->groups(['api', 'storage'])
->label('scope', 'files.read')
->label('usage.metric', 'files.{scope}.requests.read')
->label('usage.params', ['bucketId:{request.bucketId}'])
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'storage')
->label('sdk.method', 'getFileView')
@ -1255,8 +1238,6 @@ App::put('/v1/storage/buckets/:bucketId/files/:fileId')
->label('event', 'buckets.[bucketId].files.[fileId].update')
->label('audits.event', 'file.update')
->label('audits.resource', 'file/{response.$id}')
->label('usage.metric', 'files.{scope}.requests.update')
->label('usage.params', ['bucketId:{request.bucketId}'])
->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}')
->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT)
->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT)
@ -1358,8 +1339,6 @@ App::delete('/v1/storage/buckets/:bucketId/files/:fileId')
->label('event', 'buckets.[bucketId].files.[fileId].delete')
->label('audits.event', 'file.delete')
->label('audits.resource', 'file/{request.fileId}')
->label('usage.metric', 'files.{scope}.requests.delete')
->label('usage.params', ['bucketId:{request.bucketId}'])
->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}')
->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT)
->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT)
@ -1446,118 +1425,8 @@ App::delete('/v1/storage/buckets/:bucketId/files/:fileId')
$response->noContent();
});
App::get('/v1/storage/usage')
->desc('Get usage stats for storage')
->groups(['api', 'storage'])
->label('scope', 'files.read')
->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
->label('sdk.namespace', 'storage')
->label('sdk.method', 'getUsage')
->label('sdk.response.code', Response::STATUS_CODE_OK)
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
->label('sdk.response.model', Response::MODEL_USAGE_STORAGE)
->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true)
->inject('response')
->inject('dbForProject')
->action(function (string $range, Response $response, Database $dbForProject) {
$usage = [];
if (App::getEnv('_APP_USAGE_STATS', 'enabled') === 'enabled') {
$periods = [
'24h' => [
'period' => '1h',
'limit' => 24,
],
'7d' => [
'period' => '1d',
'limit' => 7,
],
'30d' => [
'period' => '1d',
'limit' => 30,
],
'90d' => [
'period' => '1d',
'limit' => 90,
],
];
$metrics = [
'project.$all.storage.size',
'buckets.$all.count.total',
'buckets.$all.requests.create',
'buckets.$all.requests.read',
'buckets.$all.requests.update',
'buckets.$all.requests.delete',
'files.$all.storage.size',
'files.$all.count.total',
'files.$all.requests.create',
'files.$all.requests.read',
'files.$all.requests.update',
'files.$all.requests.delete',
];
$stats = [];
Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) {
foreach ($metrics as $metric) {
$limit = $periods[$range]['limit'];
$period = $periods[$range]['period'];
$requestDocs = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric] = [];
foreach ($requestDocs as $requestDoc) {
$stats[$metric][] = [
'value' => $requestDoc->getAttribute('value'),
'date' => $requestDoc->getAttribute('time'),
];
}
// backfill metrics with empty values for graphs
$backfill = $limit - \count($requestDocs);
while ($backfill > 0) {
$last = $limit - $backfill - 1; // array index of last added metric
$diff = match ($period) { // convert period to seconds for unix timestamp math
'1h' => 3600,
'1d' => 86400,
};
$stats[$metric][] = [
'value' => 0,
'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)),
];
$backfill--;
}
$stats[$metric] = array_reverse($stats[$metric]);
}
});
$usage = new Document([
'range' => $range,
'bucketsCount' => $stats['buckets.$all.count.total'],
'bucketsCreate' => $stats['buckets.$all.requests.create'],
'bucketsRead' => $stats['buckets.$all.requests.read'],
'bucketsUpdate' => $stats['buckets.$all.requests.update'],
'bucketsDelete' => $stats['buckets.$all.requests.delete'],
'storage' => $stats['project.$all.storage.size'],
'filesCount' => $stats['files.$all.count.total'],
'filesCreate' => $stats['files.$all.requests.create'],
'filesRead' => $stats['files.$all.requests.read'],
'filesUpdate' => $stats['files.$all.requests.update'],
'filesDelete' => $stats['files.$all.requests.delete'],
]);
}
$response->dynamic($usage, Response::MODEL_USAGE_STORAGE);
});
App::get('/v1/storage/:bucketId/usage')
->desc('Get usage stats for a storage bucket')
->desc('Get usage stats for storage bucket')
->groups(['api', 'storage'])
->label('scope', 'files.read')
->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
@ -1578,86 +1447,124 @@ App::get('/v1/storage/:bucketId/usage')
throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND);
}
$usage = [];
if (App::getEnv('_APP_USAGE_STATS', 'enabled') === 'enabled') {
$periods = [
'24h' => [
'period' => '1h',
'limit' => 24,
],
'7d' => [
'period' => '1d',
'limit' => 7,
],
'30d' => [
'period' => '1d',
'limit' => 30,
],
'90d' => [
'period' => '1d',
'limit' => 90,
],
];
$periods = Config::getParam('usage', []);
$stats = $usage = [];
$days = $periods[$range];
$metrics = [
str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES),
str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES_STORAGE),
];
$metrics = [
"files.{$bucketId}.count.total",
"files.{$bucketId}.storage.size",
"files.{$bucketId}.requests.create",
"files.{$bucketId}.requests.read",
"files.{$bucketId}.requests.update",
"files.{$bucketId}.requests.delete",
];
$stats = [];
Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) {
foreach ($metrics as $metric) {
$limit = $periods[$range]['limit'];
$period = $periods[$range]['period'];
$requestDocs = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric] = [];
foreach ($requestDocs as $requestDoc) {
$stats[$metric][] = [
'value' => $requestDoc->getAttribute('value'),
'date' => $requestDoc->getAttribute('time'),
];
}
// backfill metrics with empty values for graphs
$backfill = $limit - \count($requestDocs);
while ($backfill > 0) {
$last = $limit - $backfill - 1; // array index of last added metric
$diff = match ($period) { // convert period to seconds for unix timestamp math
'1h' => 3600,
'1d' => 86400,
};
$stats[$metric][] = [
'value' => 0,
'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)),
];
$backfill--;
}
$stats[$metric] = array_reverse($stats[$metric]);
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
foreach ($metrics as $metric) {
$limit = $days['limit'];
$period = $days['period'];
$results = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric] = [];
foreach ($results as $result) {
$stats[$metric][$result->getAttribute('time')] = [
'value' => $result->getAttribute('value'),
];
}
});
}
});
$usage = new Document([
'range' => $range,
'filesCount' => $stats[$metrics[0]],
'filesStorage' => $stats[$metrics[1]],
'filesCreate' => $stats[$metrics[2]],
'filesRead' => $stats[$metrics[3]],
'filesUpdate' => $stats[$metrics[4]],
'filesDelete' => $stats[$metrics[5]],
]);
$format = match ($days['period']) {
'1h' => 'Y-m-d\TH:00:00.000P',
'1d' => 'Y-m-d\T00:00:00.000P',
};
foreach ($metrics as $metric) {
$usage[$metric] = [];
$leap = time() - ($days['limit'] * $days['factor']);
while ($leap < time()) {
$leap += $days['factor'];
$formatDate = date($format, $leap);
$usage[$metric][] = [
'value' => $stats[$metric][$formatDate]['value'] ?? 0,
'date' => $formatDate,
];
}
}
$response->dynamic($usage, Response::MODEL_USAGE_BUCKETS);
$response->dynamic(new Document([
'range' => $range,
'filesTotal' => $usage[$metrics[0]],
'filesStorage' => $usage[$metrics[1]],
]), Response::MODEL_USAGE_BUCKETS);
});
App::get('/v1/storage/usage')
->desc('Get usage stats for storage')
->groups(['api', 'storage'])
->label('scope', 'files.read')
->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
->label('sdk.namespace', 'storage')
->label('sdk.method', 'getUsage')
->label('sdk.response.code', Response::STATUS_CODE_OK)
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
->label('sdk.response.model', Response::MODEL_USAGE_STORAGE)
->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true)
->inject('response')
->inject('dbForProject')
->action(function (string $range, Response $response, Database $dbForProject) {
$periods = Config::getParam('usage', []);
$stats = $usage = [];
$days = $periods[$range];
$metrics = [
METRIC_BUCKETS,
METRIC_FILES,
METRIC_FILES_STORAGE,
];
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
foreach ($metrics as $metric) {
$limit = $days['limit'];
$period = $days['period'];
$results = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric] = [];
foreach ($results as $result) {
$stats[$metric][$result->getAttribute('time')] = [
'value' => $result->getAttribute('value'),
];
}
}
});
$format = match ($days['period']) {
'1h' => 'Y-m-d\TH:00:00.000P',
'1d' => 'Y-m-d\T00:00:00.000P',
};
foreach ($metrics as $metric) {
$usage[$metric] = [];
$leap = time() - ($days['limit'] * $days['factor']);
while ($leap < time()) {
$leap += $days['factor'];
$formatDate = date($format, $leap);
$usage[$metric][] = [
'value' => $stats[$metric][$formatDate]['value'] ?? 0,
'date' => $formatDate,
];
}
}
$response->dynamic(new Document([
'range' => $range,
'bucketsTotal' => $usage[$metrics[0]],
'filesTotal' => $usage[$metrics[1]],
'filesStorage' => $usage[$metrics[2]],
]), Response::MODEL_USAGE_STORAGE);
});

155
app/controllers/api/users.php
View file

@ -90,7 +90,6 @@ App::post('/v1/users')
->label('scope', 'users.write')
->label('audits.event', 'user.create')
->label('audits.resource', 'user/{response.$id}')
->label('usage.metric', 'users.{scope}.requests.create')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'create')
@ -121,7 +120,6 @@ App::post('/v1/users/bcrypt')
->label('scope', 'users.write')
->label('audits.event', 'user.create')
->label('audits.resource', 'user/{response.$id}')
->label('usage.metric', 'users.{scope}.requests.create')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'createBcryptUser')
@ -151,7 +149,6 @@ App::post('/v1/users/md5')
->label('scope', 'users.write')
->label('audits.event', 'user.create')
->label('audits.resource', 'user/{response.$id}')
->label('usage.metric', 'users.{scope}.requests.create')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'createMD5User')
@ -181,7 +178,6 @@ App::post('/v1/users/argon2')
->label('scope', 'users.write')
->label('audits.event', 'user.create')
->label('audits.resource', 'user/{response.$id}')
->label('usage.metric', 'users.{scope}.requests.create')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'createArgon2User')
@ -211,7 +207,6 @@ App::post('/v1/users/sha')
->label('scope', 'users.write')
->label('audits.event', 'user.create')
->label('audits.resource', 'user/{response.$id}')
->label('usage.metric', 'users.{scope}.requests.create')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'createSHAUser')
@ -248,7 +243,6 @@ App::post('/v1/users/phpass')
->label('scope', 'users.write')
->label('audits.event', 'user.create')
->label('audits.resource', 'user/{response.$id}')
->label('usage.metric', 'users.{scope}.requests.create')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'createPHPassUser')
@ -278,7 +272,6 @@ App::post('/v1/users/scrypt')
->label('scope', 'users.write')
->label('audits.event', 'user.create')
->label('audits.resource', 'user/{response.$id}')
->label('usage.metric', 'users.{scope}.requests.create')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'createScryptUser')
@ -321,7 +314,6 @@ App::post('/v1/users/scrypt-modified')
->label('scope', 'users.write')
->label('audits.event', 'user.create')
->label('audits.resource', 'user/{response.$id}')
->label('usage.metric', 'users.{scope}.requests.create')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'createScryptModifiedUser')
@ -351,7 +343,6 @@ App::get('/v1/users')
->desc('List Users')
->groups(['api', 'users'])
->label('scope', 'users.read')
->label('usage.metric', 'users.{scope}.requests.read')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'list')
@ -398,7 +389,6 @@ App::get('/v1/users/:userId')
->desc('Get User')
->groups(['api', 'users'])
->label('scope', 'users.read')
->label('usage.metric', 'users.{scope}.requests.read')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'get')
@ -424,7 +414,6 @@ App::get('/v1/users/:userId/prefs')
->desc('Get User Preferences')
->groups(['api', 'users'])
->label('scope', 'users.read')
->label('usage.metric', 'users.{scope}.requests.read')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'getPrefs')
@ -452,7 +441,6 @@ App::get('/v1/users/:userId/sessions')
->desc('List User Sessions')
->groups(['api', 'users'])
->label('scope', 'users.read')
->label('usage.metric', 'users.{scope}.requests.read')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'listSessions')
@ -494,7 +482,6 @@ App::get('/v1/users/:userId/memberships')
->desc('List User Memberships')
->groups(['api', 'users'])
->label('scope', 'users.read')
->label('usage.metric', 'users.{scope}.requests.read')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'listMemberships')
@ -534,7 +521,6 @@ App::get('/v1/users/:userId/logs')
->desc('List User Logs')
->groups(['api', 'users'])
->label('scope', 'users.read')
->label('usage.metric', 'users.{scope}.requests.read')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'listLogs')
@ -620,7 +606,6 @@ App::patch('/v1/users/:userId/status')
->label('audits.event', 'user.update')
->label('audits.resource', 'user/{response.$id}')
->label('audits.userId', '{response.$id}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'updateStatus')
@ -656,7 +641,6 @@ App::patch('/v1/users/:userId/verification')
->label('scope', 'users.write')
->label('audits.event', 'verification.update')
->label('audits.resource', 'user/{response.$id}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'updateEmailVerification')
@ -692,7 +676,6 @@ App::patch('/v1/users/:userId/verification/phone')
->label('scope', 'users.write')
->label('audits.event', 'verification.update')
->label('audits.resource', 'user/{response.$id}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'updatePhoneVerification')
@ -729,7 +712,6 @@ App::patch('/v1/users/:userId/name')
->label('audits.event', 'user.update')
->label('audits.resource', 'user/{response.$id}')
->label('audits.userId', '{response.$id}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'updateName')
@ -770,7 +752,6 @@ App::patch('/v1/users/:userId/password')
->label('audits.event', 'user.update')
->label('audits.resource', 'user/{response.$id}')
->label('audits.userId', '{response.$id}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'updatePassword')
@ -812,7 +793,6 @@ App::patch('/v1/users/:userId/email')
->label('audits.event', 'user.update')
->label('audits.resource', 'user/{response.$id}')
->label('audits.userId', '{response.$id}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'updateEmail')
@ -858,7 +838,6 @@ App::patch('/v1/users/:userId/phone')
->label('scope', 'users.write')
->label('audits.event', 'user.update')
->label('audits.resource', 'user/{response.$id}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'updatePhone')
@ -904,7 +883,6 @@ App::patch('/v1/users/:userId/verification')
->label('audits.event', 'verification.update')
->label('audits.resource', 'user/{request.userId}')
->label('audits.userId', '{request.userId}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'updateEmailVerification')
@ -937,7 +915,6 @@ App::patch('/v1/users/:userId/prefs')
->groups(['api', 'users'])
->label('event', 'users.[userId].update.prefs')
->label('scope', 'users.write')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'updatePrefs')
@ -973,7 +950,6 @@ App::delete('/v1/users/:userId/sessions/:sessionId')
->label('scope', 'users.write')
->label('audits.event', 'session.delete')
->label('audits.resource', 'user/{request.userId}')
->label('usage.metric', 'sessions.{scope}.requests.delete')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'deleteSession')
@ -1017,7 +993,6 @@ App::delete('/v1/users/:userId/sessions')
->label('scope', 'users.write')
->label('audits.event', 'session.delete')
->label('audits.resource', 'user/{user.$id}')
->label('usage.metric', 'sessions.{scope}.requests.delete')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'deleteSessions')
@ -1060,7 +1035,6 @@ App::delete('/v1/users/:userId')
->label('scope', 'users.write')
->label('audits.event', 'user.delete')
->label('audits.resource', 'user/{request.userId}')
->label('usage.metric', 'users.{scope}.requests.delete')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'delete')
@ -1107,96 +1081,59 @@ App::get('/v1/users/usage')
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
->label('sdk.response.model', Response::MODEL_USAGE_USERS)
->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true)
->param('provider', '', new WhiteList(\array_merge(['email', 'anonymous'], \array_map(fn ($value) => "oauth-" . $value, \array_keys(Config::getParam('providers', [])))), true), 'Provider Name.', true)
->inject('response')
->inject('dbForProject')
->inject('register')
->action(function (string $range, string $provider, Response $response, Database $dbForProject) {
->action(function (string $range, Response $response, Database $dbForProject) {
$usage = [];
if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') {
$periods = [
'24h' => [
'period' => '1h',
'limit' => 24,
],
'7d' => [
'period' => '1d',
'limit' => 7,
],
'30d' => [
'period' => '1d',
'limit' => 30,
],
'90d' => [
'period' => '1d',
'limit' => 90,
],
];
$periods = Config::getParam('usage', []);
$stats = $usage = [];
$days = $periods[$range];
$metrics = [
METRIC_USERS,
METRIC_SESSIONS,
];
$metrics = [
'users.$all.count.total',
'users.$all.requests.create',
'users.$all.requests.read',
'users.$all.requests.update',
'users.$all.requests.delete',
'sessions.$all.requests.create',
'sessions.$all.requests.delete',
"sessions.$provider.requests.create",
];
$stats = [];
Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) {
foreach ($metrics as $metric) {
$limit = $periods[$range]['limit'];
$period = $periods[$range]['period'];
$requestDocs = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric] = [];
foreach ($requestDocs as $requestDoc) {
$stats[$metric][] = [
'value' => $requestDoc->getAttribute('value'),
'date' => $requestDoc->getAttribute('time'),
];
}
// backfill metrics with empty values for graphs
$backfill = $limit - \count($requestDocs);
while ($backfill > 0) {
$last = $limit - $backfill - 1; // array index of last added metric
$diff = match ($period) { // convert period to seconds for unix timestamp math
'1h' => 3600,
'1d' => 86400,
};
$stats[$metric][] = [
'value' => 0,
'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)),
];
$backfill--;
}
$stats[$metric] = array_reverse($stats[$metric]);
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
foreach ($metrics as $metric) {
$limit = $days['limit'];
$period = $days['period'];
$results = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric] = [];
foreach ($results as $result) {
$stats[$metric][$result->getAttribute('time')] = [
'value' => $result->getAttribute('value'),
];
}
});
}
});
$usage = new Document([
'range' => $range,
'usersCount' => $stats['users.$all.count.total'] ?? [],
'usersCreate' => $stats['users.$all.requests.create'] ?? [],
'usersRead' => $stats['users.$all.requests.read'] ?? [],
'usersUpdate' => $stats['users.$all.requests.update'] ?? [],
'usersDelete' => $stats['users.$all.requests.delete'] ?? [],
'sessionsCreate' => $stats['sessions.$all.requests.create'] ?? [],
'sessionsProviderCreate' => $stats["sessions.$provider.requests.create"] ?? [],
'sessionsDelete' => $stats['sessions.$all.requests.delete' ?? []]
]);
$format = match ($days['period']) {
'1h' => 'Y-m-d\TH:00:00.000P',
'1d' => 'Y-m-d\T00:00:00.000P',
};
foreach ($metrics as $metric) {
$usage[$metric] = [];
$leap = time() - ($days['limit'] * $days['factor']);
while ($leap < time()) {
$leap += $days['factor'];
$formatDate = date($format, $leap);
$usage[$metric][] = [
'value' => $stats[$metric][$formatDate]['value'] ?? 0,
'date' => $formatDate,
];
}
}
$response->dynamic($usage, Response::MODEL_USAGE_USERS);
$response->dynamic(new Document([
'range' => $range,
'usersTotal' => $usage[$metrics[0]],
'sessionsTotal' => $usage[$metrics[1]],
]), Response::MODEL_USAGE_USERS);
});

1
app/controllers/general.php
View file

@ -631,6 +631,7 @@ App::get('/.well-known/acme-challenge')
});
include_once __DIR__ . '/shared/api.php';
include_once __DIR__ . '/shared/api/auth.php';
foreach (Config::getParam('services', []) as $service) {
include_once $service['controller'];

1
app/controllers/mock.php
View file

@ -7,7 +7,6 @@ use Utopia\Database\Document;
use Utopia\Validator\Host;
use Appwrite\Utopia\Request;
use Appwrite\Utopia\Response;
use Appwrite\Utopia\Response\Model;
use Utopia\App;
use Utopia\Validator\ArrayList;
use Utopia\Validator\Integer;

205
app/controllers/shared/api.php
View file

@ -8,8 +8,8 @@ use Appwrite\Event\Event;
use Appwrite\Event\Func;
use Appwrite\Event\Mail;
use Appwrite\Extend\Exception;
use Appwrite\Event\Usage;
use Appwrite\Messaging\Adapter\Realtime;
use Appwrite\Usage\Stats;
use Appwrite\Utopia\Response;
use Appwrite\Utopia\Request;
use Utopia\App;
@ -17,7 +17,6 @@ use Utopia\Abuse\Abuse;
use Utopia\Abuse\Adapters\TimeLimit;
use Utopia\Cache\Adapter\Filesystem;
use Utopia\Cache\Cache;
use Utopia\CLI\Console;
use Utopia\Database\Database;
use Utopia\Database\DateTime;
use Utopia\Database\Document;
@ -49,43 +48,101 @@ $parseLabel = function (string $label, array $responsePayload, array $requestPar
return $label;
};
$databaseListener = function (string $event, Document $document, Stats $usage) {
$multiplier = 1;
$databaseListener = function (string $event, Document $document, Document $project, Usage $queueForUsage, Database $dbForProject) {
$value = 1;
if ($event === Database::EVENT_DOCUMENT_DELETE) {
$multiplier = -1;
$value = -1;
}
$collection = $document->getCollection();
switch ($collection) {
case 'users':
$usage->setParam('users.{scope}.count.total', 1 * $multiplier);
switch (true) {
case $document->getCollection() === 'teams':
$queueForUsage
->addMetric(METRIC_TEAMS, $value); // per project
break;
case 'databases':
$usage->setParam('databases.{scope}.count.total', 1 * $multiplier);
case $document->getCollection() === 'users':
$queueForUsage
->addMetric(METRIC_USERS, $value); // per project
if ($event === Database::EVENT_DOCUMENT_DELETE) {
$queueForUsage
->addReduce($document);
}
break;
case 'buckets':
$usage->setParam('buckets.{scope}.count.total', 1 * $multiplier);
case $document->getCollection() === 'sessions': // sessions
$queueForUsage
->addMetric(METRIC_SESSIONS, $value); //per project
break;
case 'deployments':
$usage->setParam('deployments.{scope}.storage.size', $document->getAttribute('size') * $multiplier);
case $document->getCollection() === 'databases': // databases
$queueForUsage
->addMetric(METRIC_DATABASES, $value); // per project
if ($event === Database::EVENT_DOCUMENT_DELETE) {
$queueForUsage
->addReduce($document);
}
break;
case str_starts_with($document->getCollection(), 'database_') && !str_contains($document->getCollection(), 'collection'): //collections
$parts = explode('_', $document->getCollection());
$databaseInternalId = $parts[1] ?? 0;
$queueForUsage
->addMetric(METRIC_COLLECTIONS, $value) // per project
->addMetric(str_replace('{databaseInternalId}', $databaseInternalId, METRIC_DATABASE_ID_COLLECTIONS), $value) // per database
;
if ($event === Database::EVENT_DOCUMENT_DELETE) {
$queueForUsage
->addReduce($document);
}
break;
case str_starts_with($document->getCollection(), 'database_') && str_contains($document->getCollection(), '_collection_'): //documents
$parts = explode('_', $document->getCollection());
$databaseInternalId = $parts[1] ?? 0;
$collectionInternalId = $parts[3] ?? 0;
$queueForUsage
->addMetric(METRIC_DOCUMENTS, $value) // per project
->addMetric(str_replace('{databaseInternalId}', $databaseInternalId, METRIC_DATABASE_ID_DOCUMENTS), $value) // per database
->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$databaseInternalId, $collectionInternalId], METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS), $value); // per collection
break;
case $document->getCollection() === 'buckets': //buckets
$queueForUsage
->addMetric(METRIC_BUCKETS, $value); // per project
if ($event === Database::EVENT_DOCUMENT_DELETE) {
$queueForUsage
->addReduce($document);
}
break;
case str_starts_with($document->getCollection(), 'bucket_'): // files
$parts = explode('_', $document->getCollection());
$bucketInternalId = $parts[1];
$queueForUsage
->addMetric(METRIC_FILES, $value) // per project
->addMetric(METRIC_FILES_STORAGE, $document->getAttribute('sizeOriginal') * $value) // per project
->addMetric(str_replace('{bucketInternalId}', $bucketInternalId, METRIC_BUCKET_ID_FILES), $value) // per bucket
->addMetric(str_replace('{bucketInternalId}', $bucketInternalId, METRIC_BUCKET_ID_FILES_STORAGE), $document->getAttribute('sizeOriginal') * $value); // per bucket
break;
case $document->getCollection() === 'functions':
$queueForUsage
->addMetric(METRIC_FUNCTIONS, $value); // per project
if ($event === Database::EVENT_DOCUMENT_DELETE) {
$queueForUsage
->addReduce($document);
}
break;
case $document->getCollection() === 'deployments':
$queueForUsage
->addMetric(METRIC_DEPLOYMENTS, $value) // per project
->addMetric(METRIC_DEPLOYMENTS_STORAGE, $document->getAttribute('size') * $value) // per project
->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$document->getAttribute('resourceType'), $document->getAttribute('resourceInternalId')], METRIC_FUNCTION_ID_DEPLOYMENTS), $value)// per function
->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$document->getAttribute('resourceType'), $document->getAttribute('resourceInternalId')], METRIC_FUNCTION_ID_DEPLOYMENTS_STORAGE), $document->getAttribute('size') * $value);// per function
break;
case $document->getCollection() === 'executions':
$queueForUsage
->addMetric(METRIC_EXECUTIONS, $value) // per project
->addMetric(str_replace('{functionInternalId}', $document->getAttribute('functionInternalId'), METRIC_FUNCTION_ID_EXECUTIONS), $value);// per function
break;
default:
if (strpos($collection, 'bucket_') === 0) {
$usage
->setParam('bucketId', $document->getAttribute('bucketId'))
->setParam('files.{scope}.storage.size', $document->getAttribute('sizeOriginal') * $multiplier)
->setParam('files.{scope}.count.total', 1 * $multiplier);
} elseif (strpos($collection, 'database_') === 0) {
$usage
->setParam('databaseId', $document->getAttribute('databaseId'));
if (strpos($collection, '_collection_') !== false) {
$usage
->setParam('collectionId', $document->getAttribute('$collectionId'))
->setParam('documents.{scope}.count.total', 1 * $multiplier);
} else {
$usage->setParam('collections.{scope}.count.total', 1 * $multiplier);
}
}
break;
}
};
@ -99,12 +156,13 @@ App::init()
->inject('user')
->inject('events')
->inject('audits')
->inject('usage')
->inject('mails')
->inject('deletes')
->inject('database')
->inject('dbForProject')
->inject('queueForUsage')
->inject('mode')
->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Event $events, Audit $audits, Stats $usage, Delete $deletes, EventDatabase $database, Database $dbForProject, string $mode) use ($databaseListener) {
->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Event $events, Audit $audits, Mail $mails, Delete $deletes, EventDatabase $database, Database $dbForProject, Usage $queueForUsage, string $mode) use ($databaseListener) {
$route = $utopia->match($request);
@ -123,11 +181,11 @@ App::init()
foreach ($abuseKeyLabel as $abuseKey) {
$timeLimit = new TimeLimit($abuseKey, $route->getLabel('abuse-limit', 0), $route->getLabel('abuse-time', 3600), $dbForProject);
$timeLimit
->setParam('{userId}', $user->getId())
->setParam('{userAgent}', $request->getUserAgent(''))
->setParam('{ip}', $request->getIP())
->setParam('{url}', $request->getHostname() . $route->getPath())
->setParam('{method}', $request->getMethod());
->setParam('{userId}', $user->getId())
->setParam('{userAgent}', $request->getUserAgent(''))
->setParam('{ip}', $request->getIP())
->setParam('{url}', $request->getHostname() . $route->getPath())
->setParam('{method}', $request->getMethod());
$timeLimitArray[] = $timeLimit;
}
@ -186,20 +244,13 @@ App::init()
->setProject($project)
->setUser($user);
$usage
->setParam('projectInternalId', $project->getInternalId())
->setParam('projectId', $project->getId())
->setParam('project.{scope}.network.requests', 1)
->setParam('httpMethod', $request->getMethod())
->setParam('project.{scope}.network.inbound', 0)
->setParam('project.{scope}.network.outbound', 0);
$deletes->setProject($project);
$database->setProject($project);
$dbForProject->on(Database::EVENT_DOCUMENT_CREATE, fn ($event, Document $document) => $databaseListener($event, $document, $usage));
$dbForProject->on(Database::EVENT_DOCUMENT_DELETE, fn ($event, Document $document) => $databaseListener($event, $document, $usage));
$dbForProject
->on(Database::EVENT_DOCUMENT_CREATE, fn ($event, $document) => $databaseListener($event, $document, $project, $queueForUsage, $dbForProject))
->on(Database::EVENT_DOCUMENT_DELETE, fn ($event, $document) => $databaseListener($event, $document, $project, $queueForUsage, $dbForProject))
;
$useCache = $route->getLabel('cache', false);
@ -361,13 +412,13 @@ App::shutdown()
->inject('project')
->inject('events')
->inject('audits')
->inject('usage')
->inject('deletes')
->inject('database')
->inject('mode')
->inject('dbForProject')
->inject('queueForFunctions')
->action(function (App $utopia, Request $request, Response $response, Document $project, Event $events, Audit $audits, Stats $usage, Delete $deletes, EventDatabase $database, string $mode, Database $dbForProject, Func $queueForFunctions) use ($parseLabel) {
->inject('queueForUsage')
->inject('mode')
->action(function (App $utopia, Request $request, Response $response, Document $project, Event $events, Audit $audits, Delete $deletes, EventDatabase $database, Database $dbForProject, Func $queueForFunctions, Usage $queueForUsage, string $mode) use ($parseLabel) {
$responsePayload = $response->getPayload();
@ -375,6 +426,7 @@ App::shutdown()
if (empty($events->getPayload())) {
$events->setPayload($responsePayload);
}
/**
* Trigger functions.
*/
@ -492,10 +544,10 @@ App::shutdown()
$key = md5($request->getURI() . implode('*', $request->getParams())) . '*' . APP_CACHE_BUSTER;
$data = json_encode([
'resourceType' => $resourceType,
'resource' => $resource,
'contentType' => $response->getContentType(),
'payload' => base64_encode($data['payload']),
'resourceType' => $resourceType,
'resource' => $resource,
'contentType' => $response->getContentType(),
'payload' => base64_encode($data['payload']),
]) ;
$signature = md5($data);
@ -523,35 +575,24 @@ App::shutdown()
}
}
if (
App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled'
&& $project->getId()
&& !empty($route->getLabel('sdk.namespace', null))
) { // Don't calculate console usage on admin mode
$metric = $route->getLabel('usage.metric', '');
$usageParams = $route->getLabel('usage.params', []);
if (!empty($metric)) {
$usage->setParam($metric, 1);
foreach ($usageParams as $param) {
$param = $parseLabel($param, $responsePayload, $requestParams, $user);
$parts = explode(':', $param);
if (count($parts) != 2) {
throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Usage params not properly set');
}
$usage->setParam($parts[0], $parts[1]);
if ($project->getId() !== 'console') {
if ($mode !== APP_MODE_ADMIN) {
$fileSize = 0;
$file = $request->getFiles('file');
if (!empty($file)) {
$fileSize = (\is_array($file['size']) && isset($file['size'][0])) ? $file['size'][0] : $file['size'];
}
$queueForUsage
->addMetric(METRIC_NETWORK_REQUESTS, 1)
->addMetric(METRIC_NETWORK_INBOUND, $request->getSize() + $fileSize)
->addMetric(METRIC_NETWORK_OUTBOUND, $response->getSize());
}
$fileSize = 0;
$file = $request->getFiles('file');
if (!empty($file)) {
$fileSize = (\is_array($file['size']) && isset($file['size'][0])) ? $file['size'][0] : $file['size'];
}
$usage
->setParam('project.{scope}.network.inbound', $request->getSize() + $fileSize)
->setParam('project.{scope}.network.outbound', $response->getSize())
->submit();
$queueForUsage
->setProject($project)
->trigger();
}
});

62
app/controllers/shared/api/auth.php Normal file
View file

@ -0,0 +1,62 @@
<?php
use Appwrite\Auth\Auth;
use Appwrite\Utopia\Request;
use Utopia\App;
use Appwrite\Extend\Exception;
use Utopia\Database\Document;
use Utopia\Database\Validator\Authorization;
App::init()
->groups(['auth'])
->inject('utopia')
->inject('request')
->inject('project')
->action(function (App $utopia, Request $request, Document $project) {
$route = $utopia->match($request);
$isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles());
$isAppUser = Auth::isAppUser(Authorization::getRoles());
if ($isAppUser || $isPrivilegedUser) { // Skip limits for app and console devs
return;
}
$auths = $project->getAttribute('auths', []);
switch ($route->getLabel('auth.type', '')) {
case 'emailPassword':
if (($auths['emailPassword'] ?? true) === false) {
throw new Exception(Exception::USER_AUTH_METHOD_UNSUPPORTED, 'Email / Password authentication is disabled for this project');
}
break;
case 'magic-url':
if ($project->getAttribute('usersAuthMagicURL', true) === false) {
throw new Exception(Exception::USER_AUTH_METHOD_UNSUPPORTED, 'Magic URL authentication is disabled for this project');
}
break;
case 'anonymous':
if (($auths['anonymous'] ?? true) === false) {
throw new Exception(Exception::USER_AUTH_METHOD_UNSUPPORTED, 'Anonymous authentication is disabled for this project');
}
break;
case 'invites':
if (($auths['invites'] ?? true) === false) {
throw new Exception(Exception::USER_AUTH_METHOD_UNSUPPORTED, 'Invites authentication is disabled for this project');
}
break;
case 'jwt':
if (($auths['JWT'] ?? true) === false) {
throw new Exception(Exception::USER_AUTH_METHOD_UNSUPPORTED, 'JWT authentication is disabled for this project');
}
break;
default:
throw new Exception(Exception::USER_AUTH_METHOD_UNSUPPORTED, 'Unsupported authentication route');
break;
}
});

73
app/init.php
View file

@ -18,6 +18,7 @@ ini_set('display_startup_errors', 1);
ini_set('default_socket_timeout', -1);
error_reporting(E_ALL);
use Appwrite\Event\Usage;
use Appwrite\Extend\Exception;
use Appwrite\Auth\Auth;
use Appwrite\Event\Audit;
@ -31,11 +32,10 @@ use Appwrite\Network\Validator\Email;
use Appwrite\Network\Validator\Origin;
use Appwrite\OpenSSL\OpenSSL;
use Appwrite\URL\URL as AppwriteURL;
use Appwrite\Usage\Stats;
use Utopia\App;
use Utopia\Database\Helpers\ID;
use Utopia\Logger\Logger;
use Utopia\Config\Config;
use Utopia\Database\Helpers\ID;
use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\Database\Query;
@ -180,6 +180,39 @@ const APP_AUTH_TYPE_KEY = 'Key';
const APP_AUTH_TYPE_ADMIN = 'Admin';
// Response related
const MAX_OUTPUT_CHUNK_SIZE = 2 * 1024 * 1024; // 2MB
// Usage metrics
const METRIC_TEAMS = 'teams';
const METRIC_USERS = 'users';
const METRIC_SESSIONS = 'sessions';
const METRIC_DATABASES = 'databases';
const METRIC_COLLECTIONS = 'collections';
const METRIC_DATABASE_ID_COLLECTIONS = '{databaseInternalId}.collections';
const METRIC_DOCUMENTS = 'documents';
const METRIC_DATABASE_ID_DOCUMENTS = '{databaseInternalId}.documents';
const METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS = '{databaseInternalId}.{collectionInternalId}.documents';
const METRIC_BUCKETS = 'buckets';
const METRIC_FILES = 'files';
const METRIC_FILES_STORAGE = 'files.storage';
const METRIC_BUCKET_ID_FILES = '{bucketInternalId}.files';
const METRIC_BUCKET_ID_FILES_STORAGE = '{bucketInternalId}.files.storage';
const METRIC_FUNCTIONS = 'functions';
const METRIC_DEPLOYMENTS = 'deployments';
const METRIC_DEPLOYMENTS_STORAGE = 'deployments.storage';
const METRIC_BUILDS = 'builds';
const METRIC_BUILDS_STORAGE = 'builds.storage';
const METRIC_BUILDS_COMPUTE = 'builds.compute';
const METRIC_FUNCTION_ID_BUILDS = '{functionInternalId}.builds';
const METRIC_FUNCTION_ID_BUILDS_STORAGE = '{functionInternalId}.builds.storage';
const METRIC_FUNCTION_ID_BUILDS_COMPUTE = '{functionInternalId}.builds.compute';
const METRIC_FUNCTION_ID_DEPLOYMENTS = '{resourceType}.{resourceInternalId}.deployments';
const METRIC_FUNCTION_ID_DEPLOYMENTS_STORAGE = '{resourceType}.{resourceInternalId}.deployments.storage';
const METRIC_EXECUTIONS = 'executions';
const METRIC_EXECUTIONS_COMPUTE = 'executions.compute';
const METRIC_FUNCTION_ID_EXECUTIONS = '{functionInternalId}.executions';
const METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE = '{functionInternalId}.executions.compute';
const METRIC_NETWORK_REQUESTS = 'network.requests';
const METRIC_NETWORK_INBOUND = 'network.inbound';
const METRIC_NETWORK_OUTBOUND = 'network.outbound';
$register = new Registry();
@ -195,6 +228,7 @@ Config::load('providers', __DIR__ . '/config/providers.php');
Config::load('platforms', __DIR__ . '/config/platforms.php');
Config::load('collections', __DIR__ . '/config/collections.php');
Config::load('runtimes', __DIR__ . '/config/runtimes.php');
Config::load('usage', __DIR__ . '/config/usage.php');
Config::load('roles', __DIR__ . '/config/roles.php'); // User roles and scopes
Config::load('scopes', __DIR__ . '/config/scopes.php'); // User roles and scopes
Config::load('services', __DIR__ . '/config/services.php'); // List of services
@ -582,13 +616,11 @@ $register->set('pools', function () {
$schemes = $connection['schemes'] ?? [];
$config = [];
$dsns = explode(',', $connection['dsns'] ?? '');
foreach ($dsns as &$dsn) {
$dsn = explode('=', $dsn);
$name = ($multipe) ? $key . '_' . $dsn[0] : $key;
$dsn = $dsn[1] ?? '';
$config[] = $name;
if (empty($dsn)) {
//throw new Exception(Exception::GENERAL_SERVER_ERROR, "Missing value for DSN connection in {$key}");
continue;
@ -613,7 +645,6 @@ $register->set('pools', function () {
*
* Resource assignment to an adapter will happen below.
*/
switch ($dsnScheme) {
case 'mysql':
case 'mariadb':
@ -651,7 +682,6 @@ $register->set('pools', function () {
$pool = new Pool($name, $poolSize, function () use ($type, $resource, $dsn) {
// Get Adapter
$adapter = null;
switch ($type) {
case 'database':
$adapter = match ($dsn->getScheme()) {
@ -695,30 +725,6 @@ $register->set('pools', function () {
return $group;
});
$register->set('influxdb', function () {
// Register DB connection
$host = App::getEnv('_APP_INFLUXDB_HOST', '');
$port = App::getEnv('_APP_INFLUXDB_PORT', '');
if (empty($host) || empty($port)) {
return;
}
$driver = new InfluxDB\Driver\Curl("http://{$host}:{$port}");
$client = new InfluxDB\Client($host, $port, '', '', false, false, 5);
$client->setDriver($driver);
return $client;
});
$register->set('statsd', function () {
// Register DB connection
$host = App::getEnv('_APP_STATSD_HOST', 'telegraf');
$port = App::getEnv('_APP_STATSD_PORT', 8125);
$connection = new \Domnikl\Statsd\Connection\UdpSocket($host, $port);
$statsd = new \Domnikl\Statsd\Client($connection);
return $statsd;
});
$register->set('smtp', function () {
$mail = new PHPMailer(true);
@ -852,7 +858,6 @@ App::setResource('loggerBreadcrumbs', function () {
});
App::setResource('register', fn() => $register);
App::setResource('locale', fn() => new Locale(App::getEnv('_APP_LOCALE', 'en')));
// Queues
@ -868,9 +873,9 @@ App::setResource('queue', function (Group $pools) {
App::setResource('queueForFunctions', function (Connection $queue) {
return new Func($queue);
}, ['queue']);
App::setResource('usage', function ($register) {
return new Stats($register->get('statsd'));
}, ['register']);
App::setResource('queueForUsage', function (Connection $queue) {
return new Usage($queue);
}, ['queue']);
App::setResource('clients', function ($request, $console, $project) {
$console->setAttribute('platforms', [ // Always allow current host
'$collection' => ID::custom('platforms'),

82
app/views/install/compose.phtml
View file

@ -80,7 +80,6 @@ services:
- mariadb
- redis
# - clamav
- influxdb
environment:
- _APP_ENV
- _APP_WORKER_PER_CORE
@ -114,8 +113,6 @@ services:
- _APP_SMTP_USERNAME
- _APP_SMTP_PASSWORD
- _APP_USAGE_STATS
- _APP_INFLUXDB_HOST
- _APP_INFLUXDB_PORT
- _APP_STORAGE_LIMIT
- _APP_STORAGE_PREVIEW_LIMIT
- _APP_STORAGE_ANTIVIRUS
@ -132,8 +129,6 @@ services:
- _APP_EXECUTOR_HOST
- _APP_LOGGING_PROVIDER
- _APP_LOGGING_CONFIG
- _APP_STATSD_HOST
- _APP_STATSD_PORT
- _APP_MAINTENANCE_INTERVAL
- _APP_MAINTENANCE_RETENTION_EXECUTION
- _APP_MAINTENANCE_RETENTION_CACHE
@ -469,35 +464,6 @@ services:
- _APP_MAINTENANCE_RETENTION_AUDIT
- _APP_MAINTENANCE_RETENTION_USAGE_HOURLY
appwrite-usage:
image: <?php echo $organization; ?>/<?php echo $image; ?>:<?php echo $version."\n"; ?>
entrypoint: usage
container_name: appwrite-usage
<<: *x-logging
restart: unless-stopped
networks:
- appwrite
depends_on:
- influxdb
- mariadb
environment:
- _APP_ENV
- _APP_OPENSSL_KEY_V1
- _APP_DB_HOST
- _APP_DB_PORT
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_USAGE_AGGREGATION_INTERVAL
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_INFLUXDB_HOST
- _APP_INFLUXDB_PORT
- _APP_LOGGING_PROVIDER
- _APP_LOGGING_CONFIG
appwrite-schedule:
image: <?php echo $organization; ?>/<?php echo $image; ?>:<?php echo $version."\n"; ?>
entrypoint: schedule
@ -582,26 +548,41 @@ services:
# volumes:
# - appwrite-uploads:/storage/uploads
influxdb:
image: appwrite/influxdb:1.5.0
container_name: appwrite-influxdb
appwrite-worker-usage:
image: <?php echo $organization; ?>/<?php echo $image; ?>:<?php echo $version."\n"; ?>
entrypoint: worker-usage
<<: *x-logging
restart: unless-stopped
container_name: appwrite-worker-usage
networks:
- appwrite
volumes:
- appwrite-influxdb:/var/lib/influxdb:rw
telegraf:
image: appwrite/telegraf:1.4.0
container_name: appwrite-telegraf
<<: *x-logging
restart: unless-stopped
networks:
- appwrite
environment:
- _APP_INFLUXDB_HOST
- _APP_INFLUXDB_PORT
- ./app:/usr/src/code/app
- ./src:/usr/src/code/src
depends_on:
- redis
- mariadb
environment:
- _APP_ENV
- _APP_WORKER_PER_CORE
- _APP_CONNECTIONS_MAX
- _APP_POOL_CLIENTS
- _APP_OPENSSL_KEY_V1
- _APP_DB_HOST
- _APP_DB_PORT
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_CONNECTIONS_DB_CONSOLE
- _APP_CONNECTIONS_DB_PROJECT
- _APP_CONNECTIONS_CACHE
- _APP_CONNECTIONS_QUEUE
- _APP_USAGE_STATS
- DOCKERHUB_PULL_USERNAME
- DOCKERHUB_PULL_PASSWORD
networks:
gateway:
@ -617,7 +598,6 @@ volumes:
appwrite-cache:
appwrite-uploads:
appwrite-certificates:
appwrite-influxdb:
appwrite-config:
appwrite-functions:
appwrite-builds:

15
app/worker.php
View file

@ -3,6 +3,7 @@
require_once __DIR__ . '/init.php';
use Appwrite\Event\Func;
use Appwrite\Event\Usage;
use Swoole\Runtime;
use Utopia\App;
use Utopia\Cache\Adapter\Sharding;
@ -85,12 +86,18 @@ Server::setResource('queueForFunctions', function (Registry $register) {
);
}, ['register']);
Server::setResource('logger', function ($register) {
return $register->get('logger');
Server::setResource('queueForUsage', function (Registry $register) {
$pools = $register->get('pools');
return new Usage(
$pools
->get('queue')
->pop()
->getResource()
);
}, ['register']);
Server::setResource('statsd', function ($register) {
return $register->get('statsd');
Server::setResource('logger', function ($register) {
return $register->get('logger');
}, ['register']);
Server::setResource('pools', function ($register) {

40
app/workers/builds.php
View file

@ -6,12 +6,11 @@ use Appwrite\Messaging\Adapter\Realtime;
use Appwrite\Resque\Worker;
use Appwrite\Utopia\Response\Model\Deployment;
use Executor\Executor;
use Appwrite\Usage\Stats;
use Utopia\Database\DateTime;
use Utopia\App;
use Utopia\CLI\Console;
use Utopia\DSN\DSN;
use Utopia\Database\Helpers\ID;
use Utopia\DSN\DSN;
use Utopia\Database\Document;
use Utopia\Config\Config;
use Utopia\Database\Validator\Authorization;
@ -57,6 +56,11 @@ class BuildsV1 extends Worker
}
}
/**
* @throws \Utopia\Database\Exception\Authorization
* @throws \Utopia\Database\Exception\Structure
* @throws Throwable
*/
protected function buildDeployment(Document $project, Document $function, Document $deployment)
{
global $register;
@ -172,8 +176,8 @@ class BuildsV1 extends Worker
try {
$response = $this->executor->createRuntime(
projectId: $project->getId(),
deploymentId: $deployment->getId(),
projectId: $project->getId(),
source: $source,
image: $runtime['image'],
remove: true,
@ -249,24 +253,20 @@ class BuildsV1 extends Worker
channels: $target['channels'],
roles: $target['roles']
);
/** Update usage stats */
if (App::getEnv('_APP_USAGE_STATS', 'enabled') === 'enabled') {
$statsd = $register->get('statsd');
$usage = new Stats($statsd);
$usage
->setParam('projectInternalId', $project->getInternalId())
->setParam('projectId', $project->getId())
->setParam('functionId', $function->getId())
->setParam('builds.{scope}.compute', 1)
->setParam('buildStatus', $build->getAttribute('status', ''))
->setParam('buildTime', $build->getAttribute('duration'))
->setParam('buildSize', $build->getAttribute('size'))
->setParam('networkRequestSize', 0)
->setParam('networkResponseSize', 0)
->submit();
}
}
/** Trigger usage queue */
$this
->getUsageQueue()
->setProject($project)
->addMetric(METRIC_BUILDS, 1) // per project
->addMetric(METRIC_BUILDS_STORAGE, $build->getAttribute('size', 0))
->addMetric(METRIC_BUILDS_COMPUTE, (int)$build->getAttribute('duration', 0) * 1000)
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS), 1) // per function
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_STORAGE), $build->getAttribute('size', 0))
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE), (int)$build->getAttribute('duration', 0) * 1000)
->trigger()
;
}
public function shutdown(): void

47
app/workers/functions.php
View file

@ -2,13 +2,12 @@
require_once __DIR__ . '/../worker.php';
use Appwrite\Event\Usage;
use Utopia\Queue\Message;
use Appwrite\Event\Event;
use Appwrite\Event\Func;
use Appwrite\Messaging\Adapter\Realtime;
use Appwrite\Usage\Stats;
use Appwrite\Utopia\Response\Model\Execution;
use Domnikl\Statsd\Client;
use Executor\Executor;
use Utopia\App;
use Utopia\CLI\Console;
@ -29,7 +28,7 @@ Server::setResource('execute', function () {
return function (
Func $queueForFunctions,
Database $dbForProject,
Client $statsd,
Usage $queueForUsage,
Document $project,
Document $function,
string $trigger,
@ -40,6 +39,7 @@ Server::setResource('execute', function () {
string $eventData = null,
string $executionId = null,
) {
$user ??= new Document();
$functionId = $function->getId();
$deploymentId = $function->getAttribute('deployment', '');
@ -97,6 +97,14 @@ Server::setResource('execute', function () {
if ($execution->isEmpty()) {
throw new Exception('Failed to create or read execution');
}
/**
* Usage
*/
$queueForUsage
->addMetric(METRIC_EXECUTIONS, 1) // per project
->addMetric(str_replace('{functionInternalId}', $function->getId(), METRIC_FUNCTION_ID_EXECUTIONS), 1); // per function
}
$execution->setAttribute('status', 'processing');
@ -203,21 +211,13 @@ Server::setResource('execute', function () {
roles: $target['roles']
);
/** Update usage stats */
if (App::getEnv('_APP_USAGE_STATS', 'enabled') === 'enabled') {
$usage = new Stats($statsd);
$usage
->setParam('projectId', $project->getId())
->setParam('projectInternalId', $project->getInternalId())
->setParam('functionId', $function->getId())
->setParam('functionInternalId', $function->getInternalId())
->setParam('executions.{scope}.compute', 1)
->setParam('executionStatus', $execution->getAttribute('status', ''))
->setParam('executionTime', $execution->getAttribute('duration'))
->setParam('networkRequestSize', 0)
->setParam('networkResponseSize', 0)
->submit();
}
/** Trigger usage queue */
$queueForUsage
->setProject($project)
->addMetric(METRIC_EXECUTIONS_COMPUTE, (int)($execution->getAttribute('duration') * 1000))// per project
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE), (int)($execution->getAttribute('duration') * 1000))
->trigger()
;
};
});
@ -225,9 +225,9 @@ $server->job()
->inject('message')
->inject('dbForProject')
->inject('queueForFunctions')
->inject('statsd')
->inject('queueForUsage')
->inject('execute')
->action(function (Message $message, Database $dbForProject, Func $queueForFunctions, Client $statsd, callable $execute) {
->action(function (Message $message, Database $dbForProject, Func $queueForFunctions, Usage $queueForUsage, callable $execute) {
$payload = $message->getPayload() ?? [];
if (empty($payload)) {
@ -269,8 +269,9 @@ $server->job()
continue;
}
Console::success('Iterating function: ' . $function->getAttribute('name'));
$execute(
statsd: $statsd,
queueForUsage: $queueForUsage,
dbForProject: $dbForProject,
project: $project,
function: $function,
@ -309,7 +310,7 @@ $server->job()
data: $data,
user: $user,
jwt: $jwt,
statsd: $statsd,
queueForUsage: $queueForUsage,
);
break;
case 'schedule':
@ -325,7 +326,7 @@ $server->job()
data: null,
user: null,
jwt: null,
statsd: $statsd,
queueForUsage: $queueForUsage,
);
break;
}

289
app/workers/usage.php Normal file
View file

@ -0,0 +1,289 @@
<?php
require_once __DIR__ . '/../worker.php';
use Swoole\Timer;
use Utopia\App;
use Utopia\Cache\Cache;
use Utopia\Database\Database;
use Utopia\Database\DateTime;
use Utopia\Database\Document;
use Utopia\Database\Exception\Duplicate;
use Utopia\Database\Validator\Authorization;
use Utopia\Queue\Message;
use Utopia\CLI\Console;
use Utopia\Queue\Server;
use Utopia\Registry\Registry;
Authorization::disable();
Authorization::setDefaultStatus(false);
$stats = [];
$periods['1h'] = 'Y-m-d H:00';
$periods['1d'] = 'Y-m-d 00:00';
//$periods['1m'] = 'Y-m-1 00:00';
$periods['inf'] = '0000-00-00 00:00';
const INFINITI_PERIOD = '_inf_';
/**
* On Documents that tied by relations like functions>deployments>build || documents>collection>database || buckets>files
* When we remove a parent document we need to deduct his children aggregation from the project scope
*/
Server::setResource('reduce', function (Cache $cache, Registry $register, $pools) {
return function ($database, $projectInternalId, Document $document, array &$metrics) use ($pools, $cache, $register): void {
try {
$dbForProject = new Database(
$pools
->get($database)
->pop()
->getResource(),
$cache
);
$dbForProject->setNamespace('_' . $projectInternalId);
switch (true) {
case $document->getCollection() === 'users': // users
$sessions = count($document->getAttribute(METRIC_SESSIONS, 0));
if (!empty($sessions)) {
$metrics[] = [
'key' => METRIC_SESSIONS,
'value' => ($sessions * -1),
];
}
break;
case $document->getCollection() === 'databases': // databases
$collections = $dbForProject->getDocument('stats', md5(INFINITI_PERIOD . str_replace('{databaseInternalId}', $document->getInternalId(), METRIC_DATABASE_ID_COLLECTIONS)));
$documents = $dbForProject->getDocument('stats', md5(INFINITI_PERIOD . str_replace('{databaseInternalId}', $document->getInternalId(), METRIC_DATABASE_ID_DOCUMENTS)));
if (!empty($collections['value'])) {
$metrics[] = [
'key' => METRIC_COLLECTIONS,
'value' => ($collections['value'] * -1),
];
}
if (!empty($documents['value'])) {
$metrics[] = [
'key' => METRIC_DOCUMENTS,
'value' => ($documents['value'] * -1),
];
}
break;
case str_starts_with($document->getCollection(), 'database_') && !str_contains($document->getCollection(), 'collection'): //collections
$parts = explode('_', $document->getCollection());
$databaseInternalId = $parts[1] ?? 0;
$documents = $dbForProject->getDocument('stats', md5(INFINITI_PERIOD . str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$databaseInternalId, $document->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS)));
if (!empty($documents['value'])) {
$metrics[] = [
'key' => METRIC_DOCUMENTS,
'value' => ($documents['value'] * -1),
];
$metrics[] = [
'key' => str_replace('{databaseInternalId}', $databaseInternalId, METRIC_DATABASE_ID_DOCUMENTS),
'value' => ($documents['value'] * -1),
];
}
break;
case $document->getCollection() === 'buckets':
$files = $dbForProject->getDocument('stats', md5(INFINITI_PERIOD . str_replace('{bucketInternalId}', $document->getInternalId(), METRIC_BUCKET_ID_FILES)));
$storage = $dbForProject->getDocument('stats', md5(INFINITI_PERIOD . str_replace('{bucketInternalId}', $document->getInternalId(), METRIC_BUCKET_ID_FILES_STORAGE)));
if (!empty($files['value'])) {
$metrics[] = [
'key' => METRIC_FILES,
'value' => ($files['value'] * -1),
];
}
if (!empty($storage['value'])) {
$metrics[] = [
'key' => METRIC_FILES_STORAGE,
'value' => ($storage['value'] * -1),
];
}
break;
case $document->getCollection() === 'functions':
$deployments = $dbForProject->getDocument('stats', md5(INFINITI_PERIOD . str_replace(['{resourceType}', '{resourceInternalId}'], ['function', $document->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS)));
$deploymentsStorage = $dbForProject->getDocument('stats', md5(INFINITI_PERIOD . str_replace(['{resourceType}', '{resourceInternalId}'], ['function', $document->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS_STORAGE)));
$builds = $dbForProject->getDocument('stats', md5(INFINITI_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS)));
$buildsStorage = $dbForProject->getDocument('stats', md5(INFINITI_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_STORAGE)));
$buildsCompute = $dbForProject->getDocument('stats', md5(INFINITI_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE)));
$executions = $dbForProject->getDocument('stats', md5(INFINITI_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS)));
$executionsCompute = $dbForProject->getDocument('stats', md5(INFINITI_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE)));
if (!empty($deployments['value'])) {
$metrics[] = [
'key' => METRIC_DEPLOYMENTS,
'value' => ($deployments['value'] * -1),
];
}
if (!empty($deploymentsStorage['value'])) {
$metrics[] = [
'key' => METRIC_DEPLOYMENTS_STORAGE,
'value' => ($deploymentsStorage['value'] * -1),
];
}
if (!empty($builds['value'])) {
$metrics[] = [
'key' => METRIC_BUILDS,
'value' => ($builds['value'] * -1),
];
}
if (!empty($buildsStorage['value'])) {
$metrics[] = [
'key' => METRIC_BUILDS_STORAGE,
'value' => ($buildsStorage['value'] * -1),
];
}
if (!empty($buildsCompute['value'])) {
$metrics[] = [
'key' => METRIC_BUILDS_COMPUTE,
'value' => ($buildsCompute['value'] * -1),
];
}
if (!empty($executions['value'])) {
$metrics[] = [
'key' => METRIC_EXECUTIONS,
'value' => ($executionsCompute['value'] * -1),
];
}
if (!empty($executionsCompute['value'])) {
$metrics[] = [
'key' => METRIC_EXECUTIONS_COMPUTE,
'value' => ($executionsCompute['value'] * -1),
];
}
break;
default:
break;
}
} catch (\Exception $e) {
console::error("[reducer] " . " {DateTime::now()} " . " {$projectInternalId} " . " {$e->getMessage()}");
} finally {
$pools->reclaim();
}
};
}, ['cache', 'register', 'pools']);
$server->job()
->inject('message')
->inject('reduce')
->action(function (Message $message, callable $reduce) use (&$stats) {
$payload = $message->getPayload() ?? [];
$project = new Document($payload['project'] ?? []);
$projectId = $project->getInternalId();
foreach ($payload['reduce'] ?? [] as $document) {
if (empty($document)) {
continue;
}
$reduce(
database: $project->getAttribute('database'),
projectInternalId: $project->getInternalId(),
document: new Document($document),
metrics: $payload['metrics'],
);
}
$stats[$projectId]['database'] = $project->getAttribute('database');
foreach ($payload['metrics'] ?? [] as $metric) {
if (!isset($stats[$projectId]['keys'][$metric['key']])) {
$stats[$projectId]['keys'][$metric['key']] = $metric['value'];
continue;
}
$stats[$projectId]['keys'][$metric['key']] += $metric['value'];
}
});
$server
->workerStart()
->inject('register')
->inject('cache')
->inject('pools')
->action(function ($register, $cache, $pools) use ($periods, &$stats) {
Timer::tick(30000, function () use ($register, $cache, $pools, $periods, &$stats) {
$offset = count($stats);
$projects = array_slice($stats, 0, $offset, true);
array_splice($stats, 0, $offset);
foreach ($projects as $projectInternalId => $project) {
try {
$dbForProject = new Database(
$pools
->get($project['database'])
->pop()
->getResource(),
$cache
);
$dbForProject->setNamespace('_' . $projectInternalId);
foreach ($project['keys'] ?? [] as $key => $value) {
if ($value == 0) {
continue;
}
foreach ($periods as $period => $format) {
$time = 'inf' === $period ? null : date($format, time());
$id = \md5("{$time}_{$period}_{$key}");
try {
$dbForProject->createDocument('stats', new Document([
'$id' => $id,
'period' => $period,
'time' => $time,
'metric' => $key,
'value' => $value,
'region' => App::getEnv('_APP_REGION', 'default'),
]));
} catch (Duplicate $th) {
if ($value < 0) {
$dbForProject->decreaseDocumentAttribute(
'stats',
$id,
'value',
abs($value)
);
} else {
$dbForProject->increaseDocumentAttribute(
'stats',
$id,
'value',
$value
);
}
}
}
}
if (!empty($project['keys'])) {
$dbForProject->createDocument('statsLogger', new Document([
'time' => DateTime::now(),
'metrics' => $project['keys'],
]));
}
} catch (\Exception $e) {
console::error("[logger] " . " {DateTime::now()} " . " {$projectInternalId} " . " {$e->getMessage()}");
} finally {
$pools->reclaim();
}
}
});
});
$server->start();

3
bin/usage
View file

@ -1,3 +0,0 @@
#!/bin/sh
php /usr/src/code/app/cli.php usage $@

3
bin/worker-usage Normal file
View file

@ -0,0 +1,3 @@
#!/bin/sh
QUEUE=v1-usage php /usr/src/code/app/workers/usage.php $@

4
composer.json
View file

@ -49,11 +49,11 @@
"utopia-php/cache": "0.8.*",
"utopia-php/cli": "0.14.*",
"utopia-php/config": "0.2.*",
"utopia-php/database": "0.30.*",
"utopia-php/queue": "0.5.*",
"utopia-php/orchestration": "0.9.*",
"utopia-php/platform": "0.3.*",
"utopia-php/pools": "0.4.*",
"utopia-php/database": "0.30.*",
"utopia-php/preloader": "0.2.*",
"utopia-php/domains": "1.1.*",
"utopia-php/framework": "0.26.*",
@ -69,11 +69,9 @@
"resque/php-resque": "1.3.6",
"matomo/device-detector": "6.0.0",
"dragonmantank/cron-expression": "3.3.1",
"influxdb/influxdb-php": "1.15.2",
"phpmailer/phpmailer": "6.6.0",
"chillerlan/php-qrcode": "4.3.3",
"adhocore/jwt": "1.1.2",
"slickdeals/statsd": "3.1.0",
"webonyx/graphql-php": "14.11.*"
},
"repositories": [

727
composer.lock generated
View file

@ -4,7 +4,7 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
"This file is @generated automatically"
],
"content-hash": "9627cdaf9a987a86cecaf7961f477e42",
"content-hash": "60f1cefcf6d210ed231acb4a99f2f20c",
"packages": [
{
"name": "adhocore/jwt",
@ -479,404 +479,6 @@
],
"time": "2022-01-18T15:43:28+00:00"
},
{
"name": "guzzlehttp/guzzle",
"version": "7.5.0",
"source": {
"type": "git",
"url": "https://github.com/guzzle/guzzle.git",
"reference": "b50a2a1251152e43f6a37f0fa053e730a67d25ba"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/guzzle/guzzle/zipball/b50a2a1251152e43f6a37f0fa053e730a67d25ba",
"reference": "b50a2a1251152e43f6a37f0fa053e730a67d25ba",
"shasum": ""
},
"require": {
"ext-json": "*",
"guzzlehttp/promises": "^1.5",
"guzzlehttp/psr7": "^1.9 || ^2.4",
"php": "^7.2.5 || ^8.0",
"psr/http-client": "^1.0",
"symfony/deprecation-contracts": "^2.2 || ^3.0"
},
"provide": {
"psr/http-client-implementation": "1.0"
},
"require-dev": {
"bamarni/composer-bin-plugin": "^1.8.1",
"ext-curl": "*",
"php-http/client-integration-tests": "^3.0",
"phpunit/phpunit": "^8.5.29 || ^9.5.23",
"psr/log": "^1.1 || ^2.0 || ^3.0"
},
"suggest": {
"ext-curl": "Required for CURL handler support",
"ext-intl": "Required for Internationalized Domain Name (IDN) support",
"psr/log": "Required for using the Log middleware"
},
"type": "library",
"extra": {
"bamarni-bin": {
"bin-links": true,
"forward-command": false
},
"branch-alias": {
"dev-master": "7.5-dev"
}
},
"autoload": {
"files": [
"src/functions_include.php"
],
"psr-4": {
"GuzzleHttp\\": "src/"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Graham Campbell",
"email": "hello@gjcampbell.co.uk",
"homepage": "https://github.com/GrahamCampbell"
},
{
"name": "Michael Dowling",
"email": "mtdowling@gmail.com",
"homepage": "https://github.com/mtdowling"
},
{
"name": "Jeremy Lindblom",
"email": "jeremeamia@gmail.com",
"homepage": "https://github.com/jeremeamia"
},
{
"name": "George Mponos",
"email": "gmponos@gmail.com",
"homepage": "https://github.com/gmponos"
},
{
"name": "Tobias Nyholm",
"email": "tobias.nyholm@gmail.com",
"homepage": "https://github.com/Nyholm"
},
{
"name": "Márk Sági-Kazár",
"email": "mark.sagikazar@gmail.com",
"homepage": "https://github.com/sagikazarmark"
},
{
"name": "Tobias Schultze",
"email": "webmaster@tubo-world.de",
"homepage": "https://github.com/Tobion"
}
],
"description": "Guzzle is a PHP HTTP client library",
"keywords": [
"client",
"curl",
"framework",
"http",
"http client",
"psr-18",
"psr-7",
"rest",
"web service"
],
"support": {
"issues": "https://github.com/guzzle/guzzle/issues",
"source": "https://github.com/guzzle/guzzle/tree/7.5.0"
},
"funding": [
{
"url": "https://github.com/GrahamCampbell",
"type": "github"
},
{
"url": "https://github.com/Nyholm",
"type": "github"
},
{
"url": "https://tidelift.com/funding/github/packagist/guzzlehttp/guzzle",
"type": "tidelift"
}
],
"time": "2022-08-28T15:39:27+00:00"
},
{
"name": "guzzlehttp/promises",
"version": "1.5.2",
"source": {
"type": "git",
"url": "https://github.com/guzzle/promises.git",
"reference": "b94b2807d85443f9719887892882d0329d1e2598"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/guzzle/promises/zipball/b94b2807d85443f9719887892882d0329d1e2598",
"reference": "b94b2807d85443f9719887892882d0329d1e2598",
"shasum": ""
},
"require": {
"php": ">=5.5"
},
"require-dev": {
"symfony/phpunit-bridge": "^4.4 || ^5.1"
},
"type": "library",
"extra": {
"branch-alias": {
"dev-master": "1.5-dev"
}
},
"autoload": {
"files": [
"src/functions_include.php"
],
"psr-4": {
"GuzzleHttp\\Promise\\": "src/"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Graham Campbell",
"email": "hello@gjcampbell.co.uk",
"homepage": "https://github.com/GrahamCampbell"
},
{
"name": "Michael Dowling",
"email": "mtdowling@gmail.com",
"homepage": "https://github.com/mtdowling"
},
{
"name": "Tobias Nyholm",
"email": "tobias.nyholm@gmail.com",
"homepage": "https://github.com/Nyholm"
},
{
"name": "Tobias Schultze",
"email": "webmaster@tubo-world.de",
"homepage": "https://github.com/Tobion"
}
],
"description": "Guzzle promises library",
"keywords": [
"promise"
],
"support": {
"issues": "https://github.com/guzzle/promises/issues",
"source": "https://github.com/guzzle/promises/tree/1.5.2"
},
"funding": [
{
"url": "https://github.com/GrahamCampbell",
"type": "github"
},
{
"url": "https://github.com/Nyholm",
"type": "github"
},
{
"url": "https://tidelift.com/funding/github/packagist/guzzlehttp/promises",
"type": "tidelift"
}
],
"time": "2022-08-28T14:55:35+00:00"
},
{
"name": "guzzlehttp/psr7",
"version": "2.4.3",
"source": {
"type": "git",
"url": "https://github.com/guzzle/psr7.git",
"reference": "67c26b443f348a51926030c83481b85718457d3d"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/guzzle/psr7/zipball/67c26b443f348a51926030c83481b85718457d3d",
"reference": "67c26b443f348a51926030c83481b85718457d3d",
"shasum": ""
},
"require": {
"php": "^7.2.5 || ^8.0",
"psr/http-factory": "^1.0",
"psr/http-message": "^1.0",
"ralouphie/getallheaders": "^3.0"
},
"provide": {
"psr/http-factory-implementation": "1.0",
"psr/http-message-implementation": "1.0"
},
"require-dev": {
"bamarni/composer-bin-plugin": "^1.8.1",
"http-interop/http-factory-tests": "^0.9",
"phpunit/phpunit": "^8.5.29 || ^9.5.23"
},
"suggest": {
"laminas/laminas-httphandlerrunner": "Emit PSR-7 responses"
},
"type": "library",
"extra": {
"bamarni-bin": {
"bin-links": true,
"forward-command": false
},
"branch-alias": {
"dev-master": "2.4-dev"
}
},
"autoload": {
"psr-4": {
"GuzzleHttp\\Psr7\\": "src/"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Graham Campbell",
"email": "hello@gjcampbell.co.uk",
"homepage": "https://github.com/GrahamCampbell"
},
{
"name": "Michael Dowling",
"email": "mtdowling@gmail.com",
"homepage": "https://github.com/mtdowling"
},
{
"name": "George Mponos",
"email": "gmponos@gmail.com",
"homepage": "https://github.com/gmponos"
},
{
"name": "Tobias Nyholm",
"email": "tobias.nyholm@gmail.com",
"homepage": "https://github.com/Nyholm"
},
{
"name": "Márk Sági-Kazár",
"email": "mark.sagikazar@gmail.com",
"homepage": "https://github.com/sagikazarmark"
},
{
"name": "Tobias Schultze",
"email": "webmaster@tubo-world.de",
"homepage": "https://github.com/Tobion"
},
{
"name": "Márk Sági-Kazár",
"email": "mark.sagikazar@gmail.com",
"homepage": "https://sagikazarmark.hu"
}
],
"description": "PSR-7 message implementation that also provides common utility methods",
"keywords": [
"http",
"message",
"psr-7",
"request",
"response",
"stream",
"uri",
"url"
],
"support": {
"issues": "https://github.com/guzzle/psr7/issues",
"source": "https://github.com/guzzle/psr7/tree/2.4.3"
},
"funding": [
{
"url": "https://github.com/GrahamCampbell",
"type": "github"
},
{
"url": "https://github.com/Nyholm",
"type": "github"
},
{
"url": "https://tidelift.com/funding/github/packagist/guzzlehttp/psr7",
"type": "tidelift"
}
],
"time": "2022-10-26T14:07:24+00:00"
},
{
"name": "influxdb/influxdb-php",
"version": "1.15.2",
"source": {
"type": "git",
"url": "https://github.com/influxdata/influxdb-php.git",
"reference": "d6e59f4f04ab9107574fda69c2cbe36671253d03"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/influxdata/influxdb-php/zipball/d6e59f4f04ab9107574fda69c2cbe36671253d03",
"reference": "d6e59f4f04ab9107574fda69c2cbe36671253d03",
"shasum": ""
},
"require": {
"guzzlehttp/guzzle": "^6.0|^7.0",
"php": "^5.5 || ^7.0 || ^8.0"
},
"require-dev": {
"dms/phpunit-arraysubset-asserts": "^0.2.1",
"phpunit/phpunit": "^9.5"
},
"suggest": {
"ext-curl": "Curl extension, needed for Curl driver",
"stefanotorresi/influxdb-php-async": "An asyncronous client for InfluxDB, implemented via ReactPHP."
},
"type": "library",
"autoload": {
"psr-4": {
"InfluxDB\\": "src/InfluxDB"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Stephen Hoogendijk",
"email": "stephen@tca0.nl"
},
{
"name": "Daniel Martinez",
"email": "danimartcas@hotmail.com"
},
{
"name": "Gianluca Arbezzano",
"email": "gianarb92@gmail.com"
}
],
"description": "InfluxDB client library for PHP",
"keywords": [
"client",
"influxdata",
"influxdb",
"influxdb class",
"influxdb client",
"influxdb library",
"time series"
],
"support": {
"issues": "https://github.com/influxdata/influxdb-php/issues",
"source": "https://github.com/influxdata/influxdb-php/tree/1.15.2"
},
"abandoned": true,
"time": "2020-12-26T17:45:17+00:00"
},
{
"name": "jean85/pretty-package-versions",
"version": "1.6.0",
@ -1263,166 +865,6 @@
],
"time": "2022-02-28T15:31:21+00:00"
},
{
"name": "psr/http-client",
"version": "1.0.1",
"source": {
"type": "git",
"url": "https://github.com/php-fig/http-client.git",
"reference": "2dfb5f6c5eff0e91e20e913f8c5452ed95b86621"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/php-fig/http-client/zipball/2dfb5f6c5eff0e91e20e913f8c5452ed95b86621",
"reference": "2dfb5f6c5eff0e91e20e913f8c5452ed95b86621",
"shasum": ""
},
"require": {
"php": "^7.0 || ^8.0",
"psr/http-message": "^1.0"
},
"type": "library",
"extra": {
"branch-alias": {
"dev-master": "1.0.x-dev"
}
},
"autoload": {
"psr-4": {
"Psr\\Http\\Client\\": "src/"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "PHP-FIG",
"homepage": "http://www.php-fig.org/"
}
],
"description": "Common interface for HTTP clients",
"homepage": "https://github.com/php-fig/http-client",
"keywords": [
"http",
"http-client",
"psr",
"psr-18"
],
"support": {
"source": "https://github.com/php-fig/http-client/tree/master"
},
"time": "2020-06-29T06:28:15+00:00"
},
{
"name": "psr/http-factory",
"version": "1.0.1",
"source": {
"type": "git",
"url": "https://github.com/php-fig/http-factory.git",
"reference": "12ac7fcd07e5b077433f5f2bee95b3a771bf61be"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/php-fig/http-factory/zipball/12ac7fcd07e5b077433f5f2bee95b3a771bf61be",
"reference": "12ac7fcd07e5b077433f5f2bee95b3a771bf61be",
"shasum": ""
},
"require": {
"php": ">=7.0.0",
"psr/http-message": "^1.0"
},
"type": "library",
"extra": {
"branch-alias": {
"dev-master": "1.0.x-dev"
}
},
"autoload": {
"psr-4": {
"Psr\\Http\\Message\\": "src/"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "PHP-FIG",
"homepage": "http://www.php-fig.org/"
}
],
"description": "Common interfaces for PSR-7 HTTP message factories",
"keywords": [
"factory",
"http",
"message",
"psr",
"psr-17",
"psr-7",
"request",
"response"
],
"support": {
"source": "https://github.com/php-fig/http-factory/tree/master"
},
"time": "2019-04-30T12:38:16+00:00"
},
{
"name": "psr/http-message",
"version": "1.0.1",
"source": {
"type": "git",
"url": "https://github.com/php-fig/http-message.git",
"reference": "f6561bf28d520154e4b0ec72be95418abe6d9363"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/php-fig/http-message/zipball/f6561bf28d520154e4b0ec72be95418abe6d9363",
"reference": "f6561bf28d520154e4b0ec72be95418abe6d9363",
"shasum": ""
},
"require": {
"php": ">=5.3.0"
},
"type": "library",
"extra": {
"branch-alias": {
"dev-master": "1.0.x-dev"
}
},
"autoload": {
"psr-4": {
"Psr\\Http\\Message\\": "src/"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "PHP-FIG",
"homepage": "http://www.php-fig.org/"
}
],
"description": "Common interface for HTTP messages",
"homepage": "https://github.com/php-fig/http-message",
"keywords": [
"http",
"http-message",
"psr",
"psr-7",
"request",
"response"
],
"support": {
"source": "https://github.com/php-fig/http-message/tree/master"
},
"time": "2016-08-06T14:39:51+00:00"
},
{
"name": "psr/log",
"version": "1.1.4",
@ -1473,50 +915,6 @@
},
"time": "2021-05-03T11:20:27+00:00"
},
{
"name": "ralouphie/getallheaders",
"version": "3.0.3",
"source": {
"type": "git",
"url": "https://github.com/ralouphie/getallheaders.git",
"reference": "120b605dfeb996808c31b6477290a714d356e822"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/ralouphie/getallheaders/zipball/120b605dfeb996808c31b6477290a714d356e822",
"reference": "120b605dfeb996808c31b6477290a714d356e822",
"shasum": ""
},
"require": {
"php": ">=5.6"
},
"require-dev": {
"php-coveralls/php-coveralls": "^2.1",
"phpunit/phpunit": "^5 || ^6.5"
},
"type": "library",
"autoload": {
"files": [
"src/getallheaders.php"
]
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Ralph Khattar",
"email": "ralph.khattar@gmail.com"
}
],
"description": "A polyfill for getallheaders.",
"support": {
"issues": "https://github.com/ralouphie/getallheaders/issues",
"source": "https://github.com/ralouphie/getallheaders/tree/develop"
},
"time": "2019-03-08T08:55:37+00:00"
},
{
"name": "resque/php-resque",
"version": "v1.3.6",
@ -1600,129 +998,6 @@
},
"time": "2020-04-16T16:39:50+00:00"
},
{
"name": "slickdeals/statsd",
"version": "3.1.0",
"source": {
"type": "git",
"url": "https://github.com/Slickdeals/statsd-php.git",
"reference": "225588a0a079e145359049f6e5e23eedb1b4c17f"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/Slickdeals/statsd-php/zipball/225588a0a079e145359049f6e5e23eedb1b4c17f",
"reference": "225588a0a079e145359049f6e5e23eedb1b4c17f",
"shasum": ""
},
"require": {
"php": ">= 7.3 || ^8"
},
"replace": {
"domnikl/statsd": "self.version"
},
"require-dev": {
"friendsofphp/php-cs-fixer": "^3.0",
"phpunit/phpunit": "^9",
"vimeo/psalm": "^4.6"
},
"type": "library",
"autoload": {
"psr-4": {
"Domnikl\\Statsd\\": "src/"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Dominik Liebler",
"email": "liebler.dominik@gmail.com"
}
],
"description": "a PHP client for statsd",
"homepage": "https://github.com/Slickdeals/statsd-php",
"keywords": [
"Metrics",
"monitoring",
"statistics",
"statsd",
"udp"
],
"support": {
"issues": "https://github.com/Slickdeals/statsd-php/issues",
"source": "https://github.com/Slickdeals/statsd-php/tree/3.1.0"
},
"time": "2021-06-04T20:33:46+00:00"
},
{
"name": "symfony/deprecation-contracts",
"version": "v3.2.0",
"source": {
"type": "git",
"url": "https://github.com/symfony/deprecation-contracts.git",
"reference": "1ee04c65529dea5d8744774d474e7cbd2f1206d3"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/symfony/deprecation-contracts/zipball/1ee04c65529dea5d8744774d474e7cbd2f1206d3",
"reference": "1ee04c65529dea5d8744774d474e7cbd2f1206d3",
"shasum": ""
},
"require": {
"php": ">=8.1"
},
"type": "library",
"extra": {
"branch-alias": {
"dev-main": "3.3-dev"
},
"thanks": {
"name": "symfony/contracts",
"url": "https://github.com/symfony/contracts"
}
},
"autoload": {
"files": [
"function.php"
]
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Nicolas Grekas",
"email": "p@tchwork.com"
},
{
"name": "Symfony Community",
"homepage": "https://symfony.com/contributors"
}
],
"description": "A generic function and convention to trigger deprecation notices",
"homepage": "https://symfony.com",
"support": {
"source": "https://github.com/symfony/deprecation-contracts/tree/v3.2.0"
},
"funding": [
{
"url": "https://symfony.com/sponsor",
"type": "custom"
},
{
"url": "https://github.com/fabpot",
"type": "github"
},
{
"url": "https://tidelift.com/funding/github/packagist/symfony/symfony",
"type": "tidelift"
}
],
"time": "2022-11-25T10:21:52+00:00"
},
{
"name": "symfony/polyfill-php80",
"version": "v1.27.0",

70
docker-compose.yml
View file

@ -54,7 +54,7 @@ services:
TESTING: true
VERSION: dev
VITE_CONSOLE_MODE: self-hosted
ports:
ports:
- 9501:80
networks:
- appwrite
@ -135,8 +135,6 @@ services:
- _APP_SMTP_USERNAME
- _APP_SMTP_PASSWORD
- _APP_USAGE_STATS
- _APP_INFLUXDB_HOST
- _APP_INFLUXDB_PORT
- _APP_STORAGE_LIMIT
- _APP_STORAGE_PREVIEW_LIMIT
- _APP_STORAGE_ANTIVIRUS
@ -153,8 +151,6 @@ services:
- _APP_EXECUTOR_HOST
- _APP_LOGGING_PROVIDER
- _APP_LOGGING_CONFIG
- _APP_STATSD_HOST
- _APP_STATSD_PORT
- _APP_MAINTENANCE_INTERVAL
- _APP_MAINTENANCE_RETENTION_EXECUTION
- _APP_MAINTENANCE_RETENTION_CACHE
@ -583,13 +579,12 @@ services:
- _APP_MAINTENANCE_RETENTION_CACHE
- _APP_MAINTENANCE_RETENTION_ABUSE
- _APP_MAINTENANCE_RETENTION_AUDIT
- _APP_MAINTENANCE_RETENTION_USAGE_HOURLY
- _APP_MAINTENANCE_RETENTION_SCHEDULES
appwrite-usage:
entrypoint: usage
appwrite-worker-usage:
entrypoint: worker-usage
<<: *x-logging
container_name: appwrite-usage
container_name: appwrite-worker-usage
image: appwrite-dev
networks:
- appwrite
@ -597,7 +592,7 @@ services:
- ./app:/usr/src/code/app
- ./src:/usr/src/code/src
depends_on:
- influxdb
- redis
- mariadb
environment:
- _APP_ENV
@ -610,9 +605,6 @@ services:
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_INFLUXDB_HOST
- _APP_INFLUXDB_PORT
- _APP_USAGE_AGGREGATION_INTERVAL
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
@ -620,8 +612,10 @@ services:
- _APP_CONNECTIONS_DB_CONSOLE
- _APP_CONNECTIONS_DB_PROJECT
- _APP_CONNECTIONS_CACHE
- _APP_LOGGING_PROVIDER
- _APP_LOGGING_CONFIG
- _APP_CONNECTIONS_QUEUE
- _APP_USAGE_STATS
- DOCKERHUB_PULL_USERNAME
- DOCKERHUB_PULL_PASSWORD
appwrite-schedule:
entrypoint: schedule
@ -734,28 +728,8 @@ services:
# - appwrite
# volumes:
# - appwrite-uploads:/storage/uploads
influxdb:
image: appwrite/influxdb:1.5.0
container_name: appwrite-influxdb
<<: *x-logging
networks:
- appwrite
volumes:
- appwrite-influxdb:/var/lib/influxdb:rw
telegraf:
image: appwrite/telegraf:1.4.0
container_name: appwrite-telegraf
<<: *x-logging
networks:
- appwrite
environment:
- _APP_INFLUXDB_HOST
- _APP_INFLUXDB_PORT
# Dev Tools Start ------------------------------------------------------------------------------------------
#
#
# The Appwrite Team uses the following tools to help debug, monitor and diagnose the Appwrite stack
#
# Here is a description of the different tools and why are we using them:
@ -764,7 +738,6 @@ services:
# RequestCatcher - An HTTP server. Catches all system https calls and displays them using a simple HTTP API. Used to debug & tests webhooks and HTTP tasks
# RedisCommander - A nice UI for exploring Redis data
# Resque - A nice UI for exploring Redis pub/sub, view the different queues workloads, pending and failed tasks
# Chronograf - A nice UI for exploring InfluxDB data
# Webgrind - A nice UI for exploring and debugging code-level stuff
maildev: # used mainly for dev tests
@ -827,25 +800,6 @@ services:
# - RESQUE_WEB_PORT=6379
# - RESQUE_WEB_HTTP_BASIC_AUTH_USER=user
# - RESQUE_WEB_HTTP_BASIC_AUTH_PASSWORD=password
# chronograf:
# image: chronograf:1.6
# container_name: appwrite-chronograf
# networks:
# - appwrite
# volumes:
# - appwrite-chronograf:/var/lib/chronograf
# ports:
# - "8888:8888"
# environment:
# - INFLUXDB_URL=http://influxdb:8086
# - KAPACITOR_URL=http://kapacitor:9092
# - AUTH_DURATION=48h
# - TOKEN_SECRET=duperduper5674829!jwt
# - GH_CLIENT_ID=d86f7145a41eacfc52cc
# - GH_CLIENT_SECRET=9e0081062367a2134e7f2ea95ba1a32d08b6c8ab
# - GH_ORGS=appwrite
# webgrind:
# image: 'jokkedk/webgrind:latest'
# volumes:
@ -881,8 +835,6 @@ volumes:
appwrite-cache:
appwrite-uploads:
appwrite-certificates:
appwrite-influxdb:
appwrite-config:
appwrite-functions:
appwrite-builds:
# appwrite-chronograf:
appwrite-config:

880
docs/specs/overview.drawio.svg
View file

File diff suppressed because one or more lines are too long
Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 75 KiB

After

Width:  |  Height:  |  Size: 57 KiB

2
docs/tutorials/add-environment-variable.md
View file

@ -12,7 +12,7 @@ This tutorial will cover how to properly add a new environment variable in Appwr
### Naming environment variable
The environment variables in Appwrite are prefixed with `_APP_`. If it belongs to a specific category, the category name is appended as `_APP_REDIS` for the Redis category. The available categories are General, Redis, MariaDB, InfluxDB, StatsD, SMTP, Storage, Functions and Maintenance. Finally, a properly describing name is given to the variable. For example, `_APP_REDIS_HOST` is an environment variable for the hostname of your Redis instance. You can find more information on available categories and existing environment variables in the [environment variables doc](https://appwrite.io/docs/environment-variables).
The environment variables in Appwrite are prefixed with `_APP_`. If it belongs to a specific category, the category name is appended as `_APP_REDIS` for the Redis category. The available categories are General, Redis, MariaDB, SMTP, Storage, Functions and Maintenance. Finally, a properly describing name is given to the variable. For example, `_APP_REDIS_HOST` is an environment variable for the hostname of your Redis instance. You can find more information on available categories and existing environment variables in the [environment variables doc](https://appwrite.io/docs/environment-variables).
### Describe new environment variable

5
docs/tutorials/add-route.md
View file

@ -128,11 +128,6 @@ App::post('/v1/storage/buckets/:bucketId/files')
->label('event', 'buckets.[bucketId].files.[fileId].create')
```
#### Usage
- usage.metric - The metric the route generates.
- usage.params - Additional parameters the metrics can have.
```php
App::post('/v1/storage/buckets/:bucketId/files')
->label('usage.metric', 'files.{scope}.requests.create')

2
docs/tutorials/multi-architecture-support.md
View file

@ -6,8 +6,6 @@ A list of Appwrite CPU architecture support status. We use this list to track th
|---|---|---|---|---|---|---|---|
| appwrite/appwrite | 🟢 | 🟢 | 🟢 | 🟢 | 🟢 | 🟢 | 🟢 |
| appwrite/mariadb | 🟢 | 🟢 | 🔴 | 🔴 | 🔴 | 🟢 | 🔴 |
| appwrite/influxdb | 🟢 | 🟢 | 🟢 | 🟢 | 🔴 | 🔴 | 🔴 |
| appwrite/telegraf | 🟢 | 🟢 | 🟢 | 🟢 | 🔴 | 🔴 | 🔴 |
| appwrite/clamav | 🟢 | 🟢 | 🟢 | 🟢 | 🔴 | 🟢 | 🟢 |
| traefik | 🟢 | 🔴 | 🟢 | 🔴 | 🟢 | 🔴 | 🔴 |
| redis | 🟢 | 🟢 | 🟢 | 🟢 | 🟢 | 🟢 | 🟢 |

1
src/Appwrite/Auth/Auth.php
View file

@ -9,7 +9,6 @@ use Appwrite\Auth\Hash\Phpass;
use Appwrite\Auth\Hash\Scrypt;
use Appwrite\Auth\Hash\Scryptmodified;
use Appwrite\Auth\Hash\Sha;
use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\Database\DateTime;
use Utopia\Database\Helpers\Role;

3
src/Appwrite/Event/Event.php
View file

@ -23,6 +23,9 @@ class Event
public const FUNCTIONS_QUEUE_NAME = 'v1-functions';
public const FUNCTIONS_CLASS_NAME = 'FunctionsV1';
public const USAGE_QUEUE_NAME = 'v1-usage';
public const USAGE_CLASS_NAME = 'UsageV1';
public const WEBHOOK_QUEUE_NAME = 'v1-webhooks';
public const WEBHOOK_CLASS_NAME = 'WebhooksV1';

64
src/Appwrite/Event/Usage.php Normal file
View file

@ -0,0 +1,64 @@
<?php
namespace Appwrite\Event;
use Utopia\Queue\Client;
use Utopia\Queue\Connection;
use Utopia\Database\Document;
class Usage extends Event
{
protected array $metrics = [];
protected array $reduce = [];
public function __construct(protected Connection $connection)
{
parent::__construct(Event::USAGE_QUEUE_NAME, Event::USAGE_CLASS_NAME);
}
/**
* Add reduce.
*
* @param Document $document
* @return self
*/
public function addReduce(Document $document): self
{
$this->reduce[] = $document;
return $this;
}
/**
* Add metric.
*
* @param string $key
* @param int $value
* @return self
*/
public function addMetric(string $key, int $value): self
{
$this->metrics[] = [
'key' => $key,
'value' => $value,
];
return $this;
}
/**
* Sends metrics to the usage worker.
*
* @return string|bool
*/
public function trigger(): string|bool
{
$client = new Client($this->queue, $this->connection);
return $client->enqueue([
'project' => $this->getProject(),
'reduce' => $this->reduce,
'metrics' => $this->metrics,
]);
}
}

12
src/Appwrite/Migration/Version/V17.php
View file

@ -123,6 +123,18 @@ class V17 extends Migration
}
break;
case 'stats':
try {
/**
* Delete 'type' attribute
*/
$this->projectDB->deleteAttribute($id, 'type');
$this->projectDB->deleteCachedCollection($id);
} catch (\Throwable $th) {
Console::warning("'type' from {$id}: {$th->getMessage()}");
}
break;
case 'schedules':
try {
/**

2
src/Appwrite/Platform/Services/Tasks.php
View file

@ -12,7 +12,6 @@ use Appwrite\Platform\Tasks\PatchCreateMissingSchedules;
use Appwrite\Platform\Tasks\SDKs;
use Appwrite\Platform\Tasks\Specs;
use Appwrite\Platform\Tasks\SSL;
use Appwrite\Platform\Tasks\Usage;
use Appwrite\Platform\Tasks\Vars;
use Appwrite\Platform\Tasks\Version;
use Appwrite\Platform\Tasks\VolumeSync;
@ -24,7 +23,6 @@ class Tasks extends Service
$this->type = self::TYPE_CLI;
$this
->addAction(Version::getName(), new Version())
->addAction(Usage::getName(), new Usage())
->addAction(Vars::getName(), new Vars())
->addAction(SSL::getName(), new SSL())
->addAction(Doctor::getName(), new Doctor())

20
src/Appwrite/Platform/Tasks/Doctor.php
View file

@ -187,26 +187,6 @@ class Doctor extends Action
Console::error('🔴 ' . str_pad("SMTP", 47, '.') . 'disconnected');
}
$host = App::getEnv('_APP_STATSD_HOST', 'telegraf');
$port = App::getEnv('_APP_STATSD_PORT', 8125);
if ($fp = @\fsockopen('udp://' . $host, $port, $errCode, $errStr, 2)) {
Console::success('🟢 ' . str_pad("StatsD", 50, '.') . 'connected');
\fclose($fp);
} else {
Console::error('🔴 ' . str_pad("StatsD", 47, '.') . 'disconnected');
}
$host = App::getEnv('_APP_INFLUXDB_HOST', '');
$port = App::getEnv('_APP_INFLUXDB_PORT', '');
if ($fp = @\fsockopen($host, $port, $errCode, $errStr, 2)) {
Console::success('🟢 ' . str_pad("InfluxDB", 50, '.') . 'connected');
\fclose($fp);
} else {
Console::error('🔴 ' . str_pad("InfluxDB", 47, '.') . 'disconnected');
}
\sleep(0.2);
Console::log('');

2
src/Appwrite/Platform/Tasks/PatchCreateMissingSchedules.php
View file

@ -8,7 +8,7 @@ use Utopia\Database\DateTime;
use Utopia\Database\Document;
use Utopia\Database\Query;
use Utopia\Database\Database;
use Utopia\Database\ID;
use Utopia\Database\Helpers\ID;
use Utopia\Database\Validator\Authorization;
class PatchCreateMissingSchedules extends Action

16
src/Appwrite/Platform/Tasks/SDKs.php
View file

@ -67,19 +67,19 @@ class SDKs extends Action
Console::info('Fetching API Spec for ' . $language['name'] . ' for ' . $platform['name'] . ' (version: ' . $version . ')');
$spec = file_get_contents(__DIR__ . '/../../../app/config/specs/swagger2-' . $version . '-' . $language['family'] . '.json');
$spec = file_get_contents(__DIR__ . '/../../../../app/config/specs/swagger2-' . $version . '-' . $language['family'] . '.json');
$cover = 'https://appwrite.io/images/github.png';
$result = \realpath(__DIR__ . '/../../../app') . '/sdks/' . $key . '-' . $language['key'];
$resultExamples = \realpath(__DIR__ . '/../../..') . '/docs/examples/' . $version . '/' . $key . '-' . $language['key'];
$target = \realpath(__DIR__ . '/../../../app') . '/sdks/git/' . $language['key'] . '/';
$readme = \realpath(__DIR__ . '/../../../docs/sdks/' . $language['key'] . '/README.md');
$result = \realpath(__DIR__ . '/../../../../app') . '/sdks/' . $key . '-' . $language['key'];
$resultExamples = \realpath(__DIR__ . '/../../../..') . '/docs/examples/' . $version . '/' . $key . '-' . $language['key'];
$target = \realpath(__DIR__ . '/../../../../app') . '/sdks/git/' . $language['key'] . '/';
$readme = \realpath(__DIR__ . '/../../../../docs/sdks/' . $language['key'] . '/README.md');
$readme = ($readme) ? \file_get_contents($readme) : '';
$gettingStarted = \realpath(__DIR__ . '/../../../docs/sdks/' . $language['key'] . '/GETTING_STARTED.md');
$gettingStarted = \realpath(__DIR__ . '/../../../../docs/sdks/' . $language['key'] . '/GETTING_STARTED.md');
$gettingStarted = ($gettingStarted) ? \file_get_contents($gettingStarted) : '';
$examples = \realpath(__DIR__ . '/../../../docs/sdks/' . $language['key'] . '/EXAMPLES.md');
$examples = \realpath(__DIR__ . '/../../../../docs/sdks/' . $language['key'] . '/EXAMPLES.md');
$examples = ($examples) ? \file_get_contents($examples) : '';
$changelog = \realpath(__DIR__ . '/../../../docs/sdks/' . $language['key'] . '/CHANGELOG.md');
$changelog = \realpath(__DIR__ . '/../../../../docs/sdks/' . $language['key'] . '/CHANGELOG.md');
$changelog = ($changelog) ? \file_get_contents($changelog) : '# Change Log';
$warning = '**This SDK is compatible with Appwrite server version ' . $version . '. For older versions, please check [previous releases](' . $language['url'] . '/releases).**';
$license = 'BSD-3-Clause';

6
src/Appwrite/Platform/Tasks/Specs.php
View file

@ -40,8 +40,8 @@ class Specs extends Action
public function action(string $version, string $mode, Registry $register): void
{
$db = $register->get('db');
$redis = $register->get('cache');
//$db = $register->get('db');
//$redis = $register->get('cache');
$appRoutes = App::getRoutes();
$response = new Response(new HttpResponse());
$mocks = ($mode === 'mocks');
@ -266,7 +266,7 @@ class Specs extends Action
continue;
}
$path = __DIR__ . '/../../../app/config/specs/' . $format . '-' . $version . '-' . $platform . '.json';
$path = __DIR__ . '/../../../../app/config/specs/' . $format . '-' . $version . '-' . $platform . '.json';
if (!file_put_contents($path, json_encode($specs->parse()))) {
throw new Exception('Failed to save spec file: ' . $path);

60
src/Appwrite/Platform/Tasks/Usage.php
View file

@ -1,60 +0,0 @@
<?php
namespace Appwrite\Platform\Tasks;
use Appwrite\Usage\Calculators\TimeSeries;
use InfluxDB\Database as InfluxDatabase;
use Utopia\App;
use Utopia\CLI\Console;
use Utopia\Database\Database as UtopiaDatabase;
use Throwable;
use Utopia\Platform\Action;
use Utopia\Registry\Registry;
class Usage extends Action
{
public static function getName(): string
{
return 'usage';
}
public function __construct()
{
$this
->desc('Schedules syncing data from influxdb to Appwrite console db')
->inject('dbForConsole')
->inject('influxdb')
->inject('register')
->inject('getProjectDB')
->inject('logError')
->callback(fn ($dbForConsole, $influxDB, $register, $getProjectDB, $logError) => $this->action($dbForConsole, $influxDB, $register, $getProjectDB, $logError));
}
protected function aggregateTimeseries(UtopiaDatabase $database, InfluxDatabase $influxDB, callable $logError): void
{
}
public function action(UtopiaDatabase $dbForConsole, InfluxDatabase $influxDB, Registry $register, callable $getProjectDB, callable $logError)
{
Console::title('Usage Aggregation V1');
Console::success(APP_NAME . ' usage aggregation process v1 has started');
$errorLogger = fn(Throwable $error, string $action = 'syncUsageStats') => $logError($error, "usage", $action);
$interval = (int) App::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '30'); // 30 seconds (by default)
$region = App::getEnv('region', 'default');
$usage = new TimeSeries($region, $dbForConsole, $influxDB, $getProjectDB, $register, $errorLogger);
Console::loop(function () use ($interval, $usage) {
$now = date('d-m-Y H:i:s', time());
Console::info("[{$now}] Aggregating Timeseries Usage data every {$interval} seconds");
$loopStart = microtime(true);
$usage->collect();
$loopTook = microtime(true) - $loopStart;
$now = date('d-m-Y H:i:s', time());
Console::info("[{$now}] Aggregation took {$loopTook} seconds");
}, $interval);
}
}

30
src/Appwrite/Resque/Worker.php
View file

@ -2,6 +2,7 @@
namespace Appwrite\Resque;
use Appwrite\Event\Usage;
use Exception;
use Utopia\App;
use Utopia\Cache\Cache;
@ -9,7 +10,7 @@ use Utopia\Config\Config;
use Utopia\Cache\Adapter\Sharding;
use Utopia\CLI\Console;
use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\Pools\Group;
use Utopia\Storage\Device;
use Utopia\Storage\Device\Backblaze;
use Utopia\Storage\Device\DOSpaces;
@ -141,7 +142,7 @@ abstract class Worker
global $register;
try {
$pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */
$pools = $register->get('pools'); /** @var Group $pools */
$pools->reclaim();
$this->shutdown();
@ -176,7 +177,7 @@ abstract class Worker
{
global $register;
$pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */
$pools = $register->get('pools'); /** @var Group $pools */
if ($project->isEmpty() || $project->getId() === 'console') {
return $this->getConsoleDB();
@ -214,7 +215,7 @@ abstract class Worker
{
global $register;
$pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */
$pools = $register->get('pools'); /** @var Group $pools */
$dbAdapter = $pools
->get('console')
@ -238,7 +239,7 @@ abstract class Worker
{
global $register;
$pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */
$pools = $register->get('pools'); /** @var Group $pools */
$list = Config::getParam('pools-cache', []);
$adapters = [];
@ -254,6 +255,24 @@ abstract class Worker
return new Cache(new Sharding($adapters));
}
/**
* Get usage queue
* @return Usage
* @throws Exception
*/
protected function getUsageQueue(): Usage
{
global $register;
$pools = $register->get('pools'); /** @var Group $pools */
$queue = $pools
->get('queue')
->pop()
->getResource();
return new Usage($queue);
}
/**
* Get Functions Storage Device
* @param string $projectId of the project
@ -297,7 +316,6 @@ abstract class Worker
public function getDevice(string $root): Device
{
$connection = App::getEnv('_APP_CONNECTIONS_STORAGE', '');
$acl = 'private';
$device = Storage::DEVICE_LOCAL;
$accessKey = '';

15
src/Appwrite/Usage/Calculator.php
View file

@ -1,15 +0,0 @@
<?php
namespace Appwrite\Usage;
abstract class Calculator
{
protected string $region;
public function __construct(string $region)
{
$this->region = $region;
}
abstract public function collect(): void;
}

557
src/Appwrite/Usage/Calculators/TimeSeries.php
View file

@ -1,557 +0,0 @@
<?php
namespace Appwrite\Usage\Calculators;
use Utopia\App;
use Appwrite\Usage\Calculator;
use Utopia\Database\Database;
use Utopia\Database\Document;
use InfluxDB\Database as InfluxDatabase;
use DateTime;
use Utopia\Registry\Registry;
class TimeSeries extends Calculator
{
/**
* InfluxDB
*
* @var InfluxDatabase
*/
protected InfluxDatabase $influxDB;
/**
* Utopia Database
*
* @var Database
*/
protected Database $database;
/**
* Error Handler Callback
*
* @var callable
*/
protected $errorHandler;
/**
* Callback to get project DB
*
* @var callable
*/
protected mixed $getProjectDB;
/**
* Registry
*
* @var Registry
*/
protected Registry $register;
/**
* Latest times for metric that was synced to the database
*
* @var array
*/
private array $latestTime = [];
/**
* Periods the metrics are collected for
* @var array
*/
protected array $periods = [
[
'key' => '1h',
'startTime' => '-24 hours'
],
[
'key' => '1d',
'startTime' => '-30 days'
]
];
/**
* All the metrics that we are collecting
*
* @var array
*/
protected array $metrics = [
'project.$all.network.requests' => [
'table' => 'appwrite_usage_project_{scope}_network_requests',
],
'project.$all.network.bandwidth' => [
'table' => 'appwrite_usage_project_{scope}_network_bandwidth',
],
'project.$all.network.inbound' => [
'table' => 'appwrite_usage_project_{scope}_network_inbound',
],
'project.$all.network.outbound' => [
'table' => 'appwrite_usage_project_{scope}_network_outbound',
],
/* Users service metrics */
'users.$all.requests.create' => [
'table' => 'appwrite_usage_users_{scope}_requests_create',
],
'users.$all.requests.read' => [
'table' => 'appwrite_usage_users_{scope}_requests_read',
],
'users.$all.requests.update' => [
'table' => 'appwrite_usage_users_{scope}_requests_update',
],
'users.$all.requests.delete' => [
'table' => 'appwrite_usage_users_{scope}_requests_delete',
],
'databases.$all.requests.create' => [
'table' => 'appwrite_usage_databases_{scope}_requests_create',
],
'databases.$all.requests.read' => [
'table' => 'appwrite_usage_databases_{scope}_requests_read',
],
'databases.$all.requests.update' => [
'table' => 'appwrite_usage_databases_{scope}_requests_update',
],
'databases.$all.requests.delete' => [
'table' => 'appwrite_usage_databases_{scope}_requests_delete',
],
'collections.$all.requests.create' => [
'table' => 'appwrite_usage_collections_{scope}_requests_create',
],
'collections.$all.requests.read' => [
'table' => 'appwrite_usage_collections_{scope}_requests_read',
],
'collections.$all.requests.update' => [
'table' => 'appwrite_usage_collections_{scope}_requests_update',
],
'collections.$all.requests.delete' => [
'table' => 'appwrite_usage_collections_{scope}_requests_delete',
],
'documents.$all.requests.create' => [
'table' => 'appwrite_usage_documents_{scope}_requests_create',
],
'documents.$all.requests.read' => [
'table' => 'appwrite_usage_documents_{scope}_requests_read',
],
'documents.$all.requests.update' => [
'table' => 'appwrite_usage_documents_{scope}_requests_update',
],
'documents.$all.requests.delete' => [
'table' => 'appwrite_usage_documents_{scope}_requests_delete',
],
'collections.databaseId.requests.create' => [
'table' => 'appwrite_usage_collections_{scope}_requests_create',
'groupBy' => ['databaseId'],
],
'collections.databaseId.requests.read' => [
'table' => 'appwrite_usage_collections_{scope}_requests_read',
'groupBy' => ['databaseId'],
],
'collections.databaseId.requests.update' => [
'table' => 'appwrite_usage_collections_{scope}_requests_update',
'groupBy' => ['databaseId'],
],
'collections.databaseId.requests.delete' => [
'table' => 'appwrite_usage_collections_{scope}_requests_delete',
'groupBy' => ['databaseId'],
],
'documents.databaseId.requests.create' => [
'table' => 'appwrite_usage_documents_{scope}_requests_create',
'groupBy' => ['databaseId'],
],
'documents.databaseId.requests.read' => [
'table' => 'appwrite_usage_documents_{scope}_requests_read',
'groupBy' => ['databaseId'],
],
'documents.databaseId.requests.update' => [
'table' => 'appwrite_usage_documents_{scope}_requests_update',
'groupBy' => ['databaseId'],
],
'documents.databaseId.requests.delete' => [
'table' => 'appwrite_usage_documents_{scope}_requests_delete',
'groupBy' => ['databaseId'],
],
'documents.databaseId/collectionId.requests.create' => [
'table' => 'appwrite_usage_documents_{scope}_requests_create',
'groupBy' => ['databaseId', 'collectionId'],
],
'documents.databaseId/collectionId.requests.read' => [
'table' => 'appwrite_usage_documents_{scope}_requests_read',
'groupBy' => ['databaseId', 'collectionId'],
],
'documents.databaseId/collectionId.requests.update' => [
'table' => 'appwrite_usage_documents_{scope}_requests_update',
'groupBy' => ['databaseId', 'collectionId'],
],
'documents.databaseId/collectionId.requests.delete' => [
'table' => 'appwrite_usage_documents_{scope}_requests_delete',
'groupBy' => ['databaseId', 'collectionId'],
],
'buckets.$all.requests.create' => [
'table' => 'appwrite_usage_buckets_{scope}_requests_create',
],
'buckets.$all.requests.read' => [
'table' => 'appwrite_usage_buckets_{scope}_requests_read',
],
'buckets.$all.requests.update' => [
'table' => 'appwrite_usage_buckets_{scope}_requests_update',
],
'buckets.$all.requests.delete' => [
'table' => 'appwrite_usage_buckets_{scope}_requests_delete',
],
'files.$all.requests.create' => [
'table' => 'appwrite_usage_files_{scope}_requests_create',
],
'files.$all.requests.read' => [
'table' => 'appwrite_usage_files_{scope}_requests_read',
],
'files.$all.requests.update' => [
'table' => 'appwrite_usage_files_{scope}_requests_update',
],
'files.$all.requests.delete' => [
'table' => 'appwrite_usage_files_{scope}_requests_delete',
],
'files.bucketId.requests.create' => [
'table' => 'appwrite_usage_files_{scope}_requests_create',
'groupBy' => ['bucketId'],
],
'files.bucketId.requests.read' => [
'table' => 'appwrite_usage_files_{scope}_requests_read',
'groupBy' => ['bucketId'],
],
'files.bucketId.requests.update' => [
'table' => 'appwrite_usage_files_{scope}_requests_update',
'groupBy' => ['bucketId'],
],
'files.bucketId.requests.delete' => [
'table' => 'appwrite_usage_files_{scope}_requests_delete',
'groupBy' => ['bucketId'],
],
'sessions.$all.requests.create' => [
'table' => 'appwrite_usage_sessions__{scope}_requests_create',
],
'sessions.provider.requests.create' => [
'table' => 'appwrite_usage_sessions_{scope}_requests_create',
'groupBy' => ['provider'],
],
'sessions.$all.requests.delete' => [
'table' => 'appwrite_usage_sessions_{scope}_requests_delete',
],
'executions.$all.compute.total' => [
'table' => 'appwrite_usage_executions_{scope}_compute',
],
'builds.$all.compute.total' => [
'table' => 'appwrite_usage_builds_{scope}_compute',
],
'executions.$all.compute.failure' => [
'table' => 'appwrite_usage_executions_{scope}_compute',
'filters' => [
'functionStatus' => 'failed',
],
],
'builds.$all.compute.failure' => [
'table' => 'appwrite_usage_builds_{scope}_compute',
'filters' => [
'functionStatus' => 'failed',
],
],
'executions.$all.compute.success' => [
'table' => 'appwrite_usage_executions_{scope}_compute',
'filters' => [
'functionStatus' => 'success',
],
],
'builds.$all.compute.success' => [
'table' => 'appwrite_usage_builds_{scope}_compute',
'filters' => [
'functionStatus' => 'success',
],
],
'executions.functionId.compute.total' => [
'table' => 'appwrite_usage_executions_{scope}_compute',
'groupBy' => ['functionId'],
],
'builds.functionId.compute.total' => [
'table' => 'appwrite_usage_builds_{scope}_compute',
'groupBy' => ['functionId'],
],
'executions.functionId.compute.failure' => [
'table' => 'appwrite_usage_executions_{scope}_compute',
'groupBy' => ['functionId'],
'filters' => [
'functionStatus' => 'failed',
],
],
'builds.functionId.compute.failure' => [
'table' => 'appwrite_usage_builds_{scope}_compute',
'groupBy' => ['functionId'],
'filters' => [
'functionBuildStatus' => 'failed',
],
],
'executions.functionId.compute.success' => [
'table' => 'appwrite_usage_executions_{scope}_compute',
'groupBy' => ['functionId'],
'filters' => [
'functionStatus' => 'success',
],
],
'builds.functionId.compute.success' => [
'table' => 'appwrite_usage_builds_{scope}_compute',
'groupBy' => ['functionId'],
'filters' => [
'functionBuildStatus' => 'success',
],
],
// counters
'users.$all.count.total' => [
'table' => 'appwrite_usage_users_{scope}_count_total',
],
'buckets.$all.count.total' => [
'table' => 'appwrite_usage_buckets_{scope}_count_total',
],
'files.$all.count.total' => [
'table' => 'appwrite_usage_files_{scope}_count_total',
],
'files.bucketId.count.total' => [
'table' => 'appwrite_usage_files_{scope}_count_total',
'groupBy' => ['bucketId']
],
'databases.$all.count.total' => [
'table' => 'appwrite_usage_databases_{scope}_count_total',
],
'collections.$all.count.total' => [
'table' => 'appwrite_usage_collections_{scope}_count_total',
],
'documents.$all.count.total' => [
'table' => 'appwrite_usage_documents_{scope}_count_total',
],
'collections.databaseId.count.total' => [
'table' => 'appwrite_usage_collections_{scope}_count_total',
'groupBy' => ['databaseId']
],
'documents.databaseId.count.total' => [
'table' => 'appwrite_usage_documents_{scope}_count_total',
'groupBy' => ['databaseId']
],
'documents.databaseId/collectionId.count.total' => [
'table' => 'appwrite_usage_documents_{scope}_count_total',
'groupBy' => ['databaseId', 'collectionId']
],
'deployments.$all.storage.size' => [
'table' => 'appwrite_usage_deployments_{scope}_storage_size',
],
'project.$all.storage.size' => [
'table' => 'appwrite_usage_project_{scope}_storage_size',
],
'files.$all.storage.size' => [
'table' => 'appwrite_usage_files_{scope}_storage_size',
],
'files.$bucketId.storage.size' => [
'table' => 'appwrite_usage_files_{scope}_storage_size',
'groupBy' => ['bucketId']
],
'builds.$all.compute.time' => [
'table' => 'appwrite_usage_executions_{scope}_compute_time',
],
'executions.$all.compute.time' => [
'table' => 'appwrite_usage_executions_{scope}_compute_time',
],
'executions.functionId.compute.time' => [
'table' => 'appwrite_usage_executions_{scope}_compute_time',
'groupBy' => ['functionId'],
],
'builds.functionId.compute.time' => [
'table' => 'appwrite_usage_builds_{scope}_compute_time',
'groupBy' => ['functionId'],
],
'project.$all.compute.time' => [ // Built time + execution time
'table' => 'appwrite_usage_project_{scope}_compute_time',
'groupBy' => ['functionId'],
],
'deployments.$all.storage.size' => [
'table' => 'appwrite_usage_deployments_{scope}_storage_size'
],
'project.$all.storage.size' => [
'table' => 'appwrite_usage_project_{scope}_storage_size'
],
'files.$all.storage.size' => [
'table' => 'appwrite_usage_files_{scope}_storage_size'
],
'files.bucketId.storage.size' => [
'table' => 'appwrite_usage_files_{scope}_storage_size',
'groupBy' => ['bucketId']
]
];
public function __construct(string $region, Database $database, InfluxDatabase $influxDB, callable $getProjectDB, Registry $register, callable $errorHandler = null)
{
parent::__construct($region);
$this->database = $database;
$this->influxDB = $influxDB;
$this->getProjectDB = $getProjectDB;
$this->register = $register;
$this->errorHandler = $errorHandler;
}
/**
* Create or Update Mertic
* Create or update each metric in the stats collection for the given project
*
* @param string $projectId
* @param int $time
* @param string $period
* @param string $metric
* @param int $value
* @param int $type
*
* @return void
*/
private function createOrUpdateMetric(string $projectId, string $time, string $period, string $metric, int $value, int $type): void
{
$id = \md5("{$time}_{$period}_{$metric}");
$project = $this->database->getDocument('projects', $projectId);
$database = call_user_func($this->getProjectDB, $project);
try {
$document = $database->getDocument('stats', $id);
if ($document->isEmpty()) {
$database->createDocument('stats', new Document([
'$id' => $id,
'period' => $period,
'time' => $time,
'metric' => $metric,
'value' => $value,
'type' => $type,
'region' => $this->region,
]));
} else {
$database->updateDocument(
'stats',
$document->getId(),
$document->setAttribute('value', $value)
);
}
} catch (\Exception $e) { // if projects are deleted this might fail
if (is_callable($this->errorHandler)) {
call_user_func($this->errorHandler, $e, "sync_project_{$projectId}_metric_{$metric}");
} else {
throw $e;
}
}
$this->register->get('pools')->reclaim();
}
/**
* Sync From InfluxDB
* Sync stats from influxDB to stats collection in the Appwrite database
*
* @param string $metric
* @param array $options
* @param array $period
*
* @return void
*/
private function syncFromInfluxDB(string $metric, array $options, array $period): void
{
$start = DateTime::createFromFormat('U', \strtotime($period['startTime']))->format(DateTime::RFC3339);
if (!empty($this->latestTime[$metric][$period['key']])) {
$start = $this->latestTime[$metric][$period['key']];
}
$end = (new DateTime())->format(DateTime::RFC3339);
$table = $options['table']; //Which influxdb table to query for this metric
$groupBy = empty($options['groupBy']) ? '' : ', ' . implode(', ', array_map(fn($groupBy) => '"' . $groupBy . '" ', $options['groupBy'])); //Some sub level metrics may be grouped by other tags like collectionId, bucketId, etc
$filters = $options['filters'] ?? []; // Some metrics might have additional filters, like function's status
if (!empty($filters)) {
$filters = ' AND ' . implode(' AND ', array_map(fn ($filter, $value) => "\"{$filter}\"='{$value}'", array_keys($filters), array_values($filters)));
} else {
$filters = '';
}
$query = "SELECT sum(value) AS \"value\" ";
$query .= "FROM \"{$table}\" ";
$query .= "WHERE \"time\" > '{$start}' ";
$query .= "AND \"time\" < '{$end}' ";
$query .= "AND \"metric_type\"='counter' {$filters} ";
$query .= "GROUP BY time({$period['key']}), \"projectId\" {$groupBy} ";
$query .= "FILL(null)";
try {
$result = $this->influxDB->query($query);
$points = $result->getPoints();
foreach ($points as $point) {
$projectId = $point['projectId'];
if (!empty($projectId) && $projectId !== 'console') {
$metricUpdated = $metric;
if (!empty($groupBy)) {
foreach ($options['groupBy'] as $groupBy) {
$groupedBy = $point[$groupBy] ?? '';
if (empty($groupedBy)) {
continue;
}
$metricUpdated = str_replace($groupBy, $groupedBy, $metricUpdated);
}
}
$value = (!empty($point['value'])) ? $point['value'] : 0;
$this->createOrUpdateMetric(
$point['projectId'],
$point['time'],
$period['key'],
$metricUpdated,
$value,
0
);
$this->latestTime[$metric][$period['key']] = $point['time'];
}
}
} catch (\Exception $e) { // if projects are deleted this might fail
if (is_callable($this->errorHandler)) {
call_user_func($this->errorHandler, $e, "sync_metric_{$metric}_influxdb");
} else {
throw $e;
}
}
}
/**
* Collect Stats
* Collect all the stats from Influd DB to Database
*
* @return void
*/
public function collect(): void
{
foreach ($this->periods as $period) {
foreach ($this->metrics as $metric => $options) { //for each metrics
try {
$this->syncFromInfluxDB($metric, $options, $period);
} catch (\Exception $e) {
if (is_callable($this->errorHandler)) {
call_user_func($this->errorHandler, $e);
} else {
throw $e;
}
}
}
}
}
}

227
src/Appwrite/Usage/Stats.php
View file

@ -1,227 +0,0 @@
<?php
namespace Appwrite\Usage;
use Utopia\App;
class Stats
{
/**
* @var array
*/
protected $params = [];
/**
* @var mixed
*/
protected $statsd;
/**
* @var string
*/
protected $namespace = 'appwrite.usage';
/**
* Event constructor.
*
* @param mixed $statsd
*/
public function __construct($statsd)
{
$this->statsd = $statsd;
}
/**
* @param string $key
* @param mixed $value
*
* @return $this
*/
public function setParam(string $key, $value): self
{
$this->params[$key] = $value;
return $this;
}
/**
* @param string $key
*
* @return mixed|null
*/
public function getParam(string $key)
{
return (isset($this->params[$key])) ? $this->params[$key] : null;
}
/**
* @param string $namespace
*
* @return $this
*/
public function setNamespace(string $namespace): self
{
$this->namespace = $namespace;
return $this;
}
/**
* @return string
*/
public function getNamespace()
{
return $this->namespace;
}
/**
* Submit data to StatsD.
* Send various metrics to StatsD based on the parameters that are set
* @return void
*/
public function submit(): void
{
$projectId = $this->params['projectId'] ?? '';
$projectInternalId = $this->params['projectInternalId'];
$tags = ",projectInternalId={$projectInternalId},projectId={$projectId},version=" . App::getEnv('_APP_VERSION', 'UNKNOWN');
// the global namespace is prepended to every key (optional)
$this->statsd->setNamespace($this->namespace);
$httpRequest = $this->params['project.{scope}.network.requests'] ?? 0;
$httpMethod = $this->params['httpMethod'] ?? '';
if ($httpRequest >= 1) {
$this->statsd->increment('project.{scope}.network.requests' . $tags . ',method=' . \strtolower($httpMethod));
}
$inbound = $this->params['project.{scope}.network.inbound'] ?? 0;
$outbound = $this->params['project.{scope}.network.outbound'] ?? 0;
$this->statsd->count('project.{scope}.network.inbound' . $tags, $inbound);
$this->statsd->count('project.{scope}.network.outbound' . $tags, $outbound);
$this->statsd->count('project.{scope}.network.bandwidth' . $tags, $inbound + $outbound);
$usersMetrics = [
'users.{scope}.requests.create',
'users.{scope}.requests.read',
'users.{scope}.requests.update',
'users.{scope}.requests.delete',
'users.{scope}.count.total',
];
foreach ($usersMetrics as $metric) {
$value = $this->params[$metric] ?? 0;
if ($value === 1 || $value === -1) {
$this->statsd->count($metric . $tags, $value);
}
}
$dbMetrics = [
'databases.{scope}.requests.create',
'databases.{scope}.requests.read',
'databases.{scope}.requests.update',
'databases.{scope}.requests.delete',
'collections.{scope}.requests.create',
'collections.{scope}.requests.read',
'collections.{scope}.requests.update',
'collections.{scope}.requests.delete',
'documents.{scope}.requests.create',
'documents.{scope}.requests.read',
'documents.{scope}.requests.update',
'documents.{scope}.requests.delete',
'databases.{scope}.count.total',
'collections.{scope}.count.total',
'documents.{scope}.count.total'
];
foreach ($dbMetrics as $metric) {
$value = $this->params[$metric] ?? 0;
if ($value === 1 || $value === -1) {
$dbTags = $tags . ",collectionId=" . ($this->params['collectionId'] ?? '') . ",databaseId=" . ($this->params['databaseId'] ?? '');
$this->statsd->count($metric . $dbTags, $value);
}
}
$storageMertics = [
'buckets.{scope}.requests.create',
'buckets.{scope}.requests.read',
'buckets.{scope}.requests.update',
'buckets.{scope}.requests.delete',
'files.{scope}.requests.create',
'files.{scope}.requests.read',
'files.{scope}.requests.update',
'files.{scope}.requests.delete',
'buckets.{scope}.count.total',
'files.{scope}.count.total',
'files.{scope}.storage.size'
];
foreach ($storageMertics as $metric) {
$value = $this->params[$metric] ?? 0;
if ($value !== 0) {
$storageTags = $tags . ",bucketId=" . ($this->params['bucketId'] ?? '');
$this->statsd->count($metric . $storageTags, $value);
}
}
$sessionsMetrics = [
'sessions.{scope}.requests.create',
'sessions.{scope}.requests.update',
'sessions.{scope}.requests.delete',
];
foreach ($sessionsMetrics as $metric) {
$value = $this->params[$metric] ?? 0;
if ($value >= 1) {
$sessionTags = $tags . ",provider=" . ($this->params['provider'] ?? '');
$this->statsd->count($metric . $sessionTags, $value);
}
}
$functionId = $this->params['functionId'] ?? '';
$functionExecution = $this->params['executions.{scope}.compute'] ?? 0;
$functionExecutionTime = ($this->params['executionTime'] ?? 0) * 1000; // ms
$functionExecutionStatus = $this->params['executionStatus'] ?? '';
$functionBuild = $this->params['builds.{scope}.compute'] ?? 0;
$functionBuildTime = ($this->params['buildTime'] ?? 0) * 1000; // ms
$functionBuildSize = ($this->params['buildSize'] ?? 0); // bytes
$functionBuildStatus = $this->params['buildStatus'] ?? '';
$functionCompute = $functionExecutionTime + $functionBuildTime;
$functionTags = $tags . ',functionId=' . $functionId;
$deploymentSize = $this->params['deployment.{scope}.storage.size'] ?? 0;
$storageSize = $this->params['files.{scope}.storage.size'] ?? 0;
if ($deploymentSize + $storageSize > 0 || $deploymentSize + $storageSize <= -1) {
$this->statsd->count('project.{scope}.storage.size' . $tags, $deploymentSize + $storageSize);
}
if ($deploymentSize !== 0) {
$this->statsd->count('deployments.{scope}.storage.size' . $functionTags, $deploymentSize);
}
if ($functionExecution >= 1) {
$this->statsd->increment('executions.{scope}.compute' . $functionTags . ',functionStatus=' . $functionExecutionStatus);
if ($functionExecutionTime > 0) {
$this->statsd->count('executions.{scope}.compute.time' . $functionTags, $functionExecutionTime);
}
}
if ($functionBuild >= 1) {
$this->statsd->increment('builds.{scope}.compute' . $functionTags . ',functionBuildStatus=' . $functionBuildStatus);
$this->statsd->count('builds.{scope}.compute.time' . $functionTags, $functionBuildTime);
$this->statsd->count('builds.{scope}.storage.size' . $functionTags, $functionBuildSize);
}
if ($functionBuild + $functionExecution >= 1) {
$this->statsd->count('project.{scope}.compute.time' . $functionTags, $functionCompute);
}
$this->reset();
}
public function reset(): self
{
$this->params = [];
$this->namespace = 'appwrite.usage';
return $this;
}
}

30
src/Appwrite/Utopia/Response/Model/UsageBuckets.php
View file

@ -16,7 +16,7 @@ class UsageBuckets extends Model
'default' => '',
'example' => '30d',
])
->addRule('filesCount', [
->addRule('filesTotal', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for total number of files in this bucket.',
'default' => [],
@ -30,34 +30,6 @@ class UsageBuckets extends Model
'example' => [],
'array' => true
])
->addRule('filesCreate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for files created.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('filesRead', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for files read.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('filesUpdate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for files updated.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('filesDelete', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for files deleted.',
'default' => [],
'example' => [],
'array' => true
])
;
}

30
src/Appwrite/Utopia/Response/Model/UsageCollection.php
View file

@ -16,41 +16,13 @@ class UsageCollection extends Model
'default' => '',
'example' => '30d',
])
->addRule('documentsCount', [
->addRule('documentsTotal', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for total number of documents.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('documentsCreate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for documents created.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('documentsRead', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for documents read.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('documentsUpdate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for documents updated.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('documentsDelete', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for documents deleted.',
'default' => [],
'example' => [],
'array' => true
])
;
}

62
src/Appwrite/Utopia/Response/Model/UsageDatabase.php
View file

@ -16,72 +16,16 @@ class UsageDatabase extends Model
'default' => '',
'example' => '30d',
])
->addRule('documentsCount', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for total number of documents.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('collectionsCount', [
->addRule('collectionsTotal', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for total number of collections.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('documentsCreate', [
->addRule('documentsTotal', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for documents created.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('documentsRead', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for documents read.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('documentsUpdate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for documents updated.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('documentsDelete', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for documents deleted.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('collectionsCreate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for collections created.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('collectionsRead', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for collections read.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('collectionsUpdate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for collections updated.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('collectionsDelete', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for collections delete.',
'description' => 'Aggregated stats for total number of documents.',
'default' => [],
'example' => [],
'array' => true

102
src/Appwrite/Utopia/Response/Model/UsageDatabases.php
View file

@ -16,111 +16,27 @@ class UsageDatabases extends Model
'default' => '',
'example' => '30d',
])
->addRule('databasesCount', [
->addRule('databasesTotal', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for total number of documents.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('documentsCount', [
->addRule('collectionsTotal', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for total number of collections.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('documentsTotal', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for total number of documents.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('collectionsCount', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for total number of collections.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('databasesCreate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for documents created.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('databasesRead', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for documents read.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('databasesUpdate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for documents updated.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('databasesDelete', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for total number of collections.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('documentsCreate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for documents created.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('documentsRead', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for documents read.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('documentsUpdate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for documents updated.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('documentsDelete', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for documents deleted.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('collectionsCreate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for collections created.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('collectionsRead', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for collections read.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('collectionsUpdate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for collections updated.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('collectionsDelete', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for collections delete.',
'default' => [],
'example' => [],
'array' => true
])
;
}

50
src/Appwrite/Utopia/Response/Model/UsageFunction.php
View file

@ -16,30 +16,16 @@ class UsageFunction extends Model
'default' => '',
'example' => '30d',
])
->addRule('executionsTotal', [
->addRule('deploymentsTotal', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for number of function executions.',
'description' => 'Aggregated stats for number of function deployments.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('executionsFailure', [
->addRule('deploymentsStorage', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for function execution failures.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('executionsSuccess', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for function execution successes.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('executionsTime', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for function execution duration.',
'description' => 'Aggregated stats for function deployments storage.',
'default' => [],
'example' => [],
'array' => true
@ -51,23 +37,31 @@ class UsageFunction extends Model
'example' => [],
'array' => true
])
->addRule('buildsFailure', [
->addRule('buildsStorage', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for function build failures.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('buildsSuccess', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for function build successes.',
'description' => 'Aggregated stats for builds storage.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('buildsTime', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for function build duration.',
'description' => 'Aggregated stats for function build compute.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('executionsTotal', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for number of function executions.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('executionsTime', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for function execution compute.',
'default' => [],
'example' => [],
'array' => true

47
src/Appwrite/Utopia/Response/Model/UsageFunctions.php
View file

@ -16,30 +16,23 @@ class UsageFunctions extends Model
'default' => '',
'example' => '30d',
])
->addRule('executionsTotal', [
->addRule('functionsTotal', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for number of function executions.',
'description' => 'Aggregated stats for number of functions.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('executionsFailure', [
->addRule('deploymentsTotal', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for function execution failures.',
'description' => 'Aggregated stats for number of function deployments.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('executionsSuccess', [
->addRule('deploymentsStorage', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for function execution successes.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('executionsTime', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for function execution duration.',
'description' => 'Aggregated stats for function deployments storage.',
'default' => [],
'example' => [],
'array' => true
@ -51,23 +44,31 @@ class UsageFunctions extends Model
'example' => [],
'array' => true
])
->addRule('buildsFailure', [
->addRule('buildsStorage', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for function build failures.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('buildsSuccess', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for function build successes.',
'description' => 'Aggregated stats for builds storage.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('buildsTime', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for function build duration.',
'description' => 'Aggregated stats for function build compute.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('executionsTotal', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for number of function executions.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('executionsTime', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for function execution compute.',
'default' => [],
'example' => [],
'array' => true

14
src/Appwrite/Utopia/Response/Model/UsageProject.php
View file

@ -16,7 +16,7 @@ class UsageProject extends Model
'default' => '',
'example' => '30d',
])
->addRule('requests', [
->addRule('requestsTotal', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for number of requests.',
'default' => [],
@ -30,42 +30,42 @@ class UsageProject extends Model
'example' => [],
'array' => true
])
->addRule('executions', [
->addRule('executionsTotal', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for function executions.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('documents', [
->addRule('documentsTotal', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for number of documents.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('databases', [
->addRule('databasesTotal', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for number of databases.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('users', [
->addRule('usersTotal', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for number of users.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('storage', [
->addRule('filesStorage', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for the occupied storage size (in bytes).',
'default' => [],
'example' => [],
'array' => true
])
->addRule('buckets', [
->addRule('bucketsTotal', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for number of buckets.',
'default' => [],

66
src/Appwrite/Utopia/Response/Model/UsageStorage.php
View file

@ -16,79 +16,23 @@ class UsageStorage extends Model
'default' => '',
'example' => '30d',
])
->addRule('storage', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for the occupied storage size (in bytes).',
'default' => [],
'example' => [],
'array' => true
])
->addRule('filesCount', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for total number of files.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('bucketsCount', [
->addRule('bucketsTotal', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for total number of buckets.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('bucketsCreate', [
->addRule('filesTotal', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for buckets created.',
'description' => 'Aggregated stats for total number of files.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('bucketsRead', [
->addRule('filesStorage', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for buckets read.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('bucketsUpdate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for buckets updated.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('bucketsDelete', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for buckets deleted.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('filesCreate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for files created.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('filesRead', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for files read.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('filesUpdate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for files updated.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('filesDelete', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for files deleted.',
'description' => 'Aggregated stats for the occupied storage size (in bytes).',
'default' => [],
'example' => [],
'array' => true

47
src/Appwrite/Utopia/Response/Model/UsageUsers.php
View file

@ -16,62 +16,21 @@ class UsageUsers extends Model
'default' => '',
'example' => '30d',
])
->addRule('usersCount', [
->addRule('usersTotal', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for total number of users.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('usersCreate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for users created.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('usersRead', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for users read.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('usersUpdate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for users updated.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('usersDelete', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for users deleted.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('sessionsCreate', [
->addRule('sessionsTotal', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for sessions created.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('sessionsProviderCreate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for sessions created for a provider ( email, anonymous or oauth2 ).',
'default' => [],
'example' => [],
'array' => true
])
->addRule('sessionsDelete', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for sessions deleted.',
'default' => [],
'example' => [],
'array' => true
])
;
}

1
src/Executor/Executor.php
View file

@ -4,7 +4,6 @@ namespace Executor;
use Exception;
use Utopia\App;
use Utopia\CLI\Console;
class Executor
{

781
tests/e2e/General/UsageTest.php
View file

File diff suppressed because it is too large Load diff

37
tests/e2e/Services/Account/AccountBase.php
View file

@ -35,8 +35,7 @@ trait AccountBase
$this->assertEquals($response['headers']['status-code'], 201);
$this->assertNotEmpty($response['body']);
$this->assertNotEmpty($response['body']['$id']);
$dateValidator = new DatetimeValidator();
$this->assertEquals(true, $dateValidator->isValid($response['body']['registration']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($response['body']['registration']));
$this->assertEquals($response['body']['email'], $email);
$this->assertEquals($response['body']['name'], $name);
@ -200,8 +199,7 @@ trait AccountBase
$this->assertEquals($response['headers']['status-code'], 200);
$this->assertNotEmpty($response['body']);
$this->assertNotEmpty($response['body']['$id']);
$dateValidator = new DatetimeValidator();
$this->assertEquals(true, $dateValidator->isValid($response['body']['registration']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($response['body']['registration']));
$this->assertEquals($response['body']['email'], $email);
$this->assertEquals($response['body']['name'], $name);
@ -346,8 +344,7 @@ trait AccountBase
$this->assertIsNumeric($response['body']['total']);
$this->assertContains($response['body']['logs'][1]['event'], ["session.create"]);
$this->assertEquals($response['body']['logs'][1]['ip'], filter_var($response['body']['logs'][1]['ip'], FILTER_VALIDATE_IP));
$dateValidator = new DatetimeValidator();
$this->assertEquals(true, $dateValidator->isValid($response['body']['logs'][1]['time']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($response['body']['logs'][1]['time']));
$this->assertEquals('Windows', $response['body']['logs'][1]['osName']);
$this->assertEquals('WIN', $response['body']['logs'][1]['osCode']);
@ -369,7 +366,7 @@ trait AccountBase
$this->assertContains($response['body']['logs'][2]['event'], ["user.create"]);
$this->assertEquals($response['body']['logs'][2]['ip'], filter_var($response['body']['logs'][2]['ip'], FILTER_VALIDATE_IP));
$this->assertEquals(true, $dateValidator->isValid($response['body']['logs'][2]['time']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($response['body']['logs'][2]['time']));
$this->assertEquals('Windows', $response['body']['logs'][2]['osName']);
$this->assertEquals('WIN', $response['body']['logs'][2]['osCode']);
@ -480,8 +477,7 @@ trait AccountBase
$this->assertIsArray($response['body']);
$this->assertNotEmpty($response['body']);
$this->assertNotEmpty($response['body']['$id']);
$dateValidator = new DatetimeValidator();
$this->assertEquals(true, $dateValidator->isValid($response['body']['registration']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($response['body']['registration']));
$this->assertEquals($response['body']['email'], $email);
$this->assertEquals($response['body']['name'], $newName);
@ -548,8 +544,7 @@ trait AccountBase
$this->assertIsArray($response['body']);
$this->assertNotEmpty($response['body']);
$this->assertNotEmpty($response['body']['$id']);
$dateValidator = new DatetimeValidator();
$this->assertEquals(true, $dateValidator->isValid($response['body']['registration']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($response['body']['registration']));
$this->assertEquals($response['body']['email'], $email);
$response = $this->client->call(Client::METHOD_POST, '/account/sessions/email', array_merge([
@ -639,8 +634,7 @@ trait AccountBase
$this->assertIsArray($response['body']);
$this->assertNotEmpty($response['body']);
$this->assertNotEmpty($response['body']['$id']);
$dateValidator = new DatetimeValidator();
$this->assertEquals(true, $dateValidator->isValid($response['body']['registration']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($response['body']['registration']));
$this->assertEquals($response['body']['email'], $newEmail);
/**
@ -682,7 +676,7 @@ trait AccountBase
$this->assertEquals($response['headers']['status-code'], 201);
$this->assertNotEmpty($response['body']);
$this->assertNotEmpty($response['body']['$id']);
$this->assertEquals(true, $dateValidator->isValid($response['body']['registration']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($response['body']['registration']));
$this->assertEquals($response['body']['email'], $data['email']);
$this->assertEquals($response['body']['name'], $data['name']);
@ -824,8 +818,7 @@ trait AccountBase
$this->assertEquals(201, $response['headers']['status-code']);
$this->assertNotEmpty($response['body']['$id']);
$this->assertEmpty($response['body']['secret']);
$dateValidator = new DatetimeValidator();
$this->assertEquals(true, $dateValidator->isValid($response['body']['expire']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($response['body']['expire']));
$lastEmail = $this->getLastEmail();
@ -1127,8 +1120,7 @@ trait AccountBase
$this->assertEquals(201, $response['headers']['status-code']);
$this->assertNotEmpty($response['body']['$id']);
$this->assertEmpty($response['body']['secret']);
$dateValidator = new DatetimeValidator();
$this->assertEquals(true, $dateValidator->isValid($response['body']['expire']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($response['body']['expire']));
$lastEmail = $this->getLastEmail();
@ -1282,8 +1274,7 @@ trait AccountBase
$this->assertEquals(201, $response['headers']['status-code']);
$this->assertNotEmpty($response['body']['$id']);
$this->assertEmpty($response['body']['secret']);
$dateValidator = new DatetimeValidator();
$this->assertEquals(true, $dateValidator->isValid($response['body']['expire']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($response['body']['expire']));
$userId = $response['body']['userId'];
@ -1389,8 +1380,7 @@ trait AccountBase
$this->assertEquals($response['headers']['status-code'], 200);
$this->assertNotEmpty($response['body']);
$this->assertNotEmpty($response['body']['$id']);
$dateValidator = new DatetimeValidator();
$this->assertEquals(true, $dateValidator->isValid($response['body']['registration']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($response['body']['registration']));
$this->assertEquals($response['body']['email'], $email);
$this->assertTrue($response['body']['emailVerification']);
@ -1450,8 +1440,7 @@ trait AccountBase
$this->assertIsArray($response['body']);
$this->assertNotEmpty($response['body']);
$this->assertNotEmpty($response['body']['$id']);
$dateValidator = new DatetimeValidator();
$this->assertEquals(true, $dateValidator->isValid($response['body']['registration']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($response['body']['registration']));
$this->assertEquals($response['body']['email'], $email);
$response = $this->client->call(Client::METHOD_POST, '/account/sessions/email', array_merge([

7
tests/e2e/Services/Account/AccountConsoleClientTest.php
View file

@ -6,9 +6,10 @@ use Appwrite\Extend\Exception;
use Tests\E2E\Scopes\Scope;
use Tests\E2E\Scopes\ProjectConsole;
use Tests\E2E\Scopes\SideClient;
use Utopia\Database\ID;
use Utopia\Database\Helpers\ID;
use Utopia\Database\DateTime;
use Tests\E2E\Client;
use Utopia\Database\Validator\DatetimeValidator;
class AccountConsoleClientTest extends Scope
{
@ -72,7 +73,7 @@ class AccountConsoleClientTest extends Scope
$this->assertEquals(201, $response['headers']['status-code']);
$this->assertNotEmpty($response['body']);
$this->assertNotEmpty($response['body']['$id']);
$this->assertEquals(true, DateTime::isValid($response['body']['registration']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($response['body']['registration']));
$this->assertEquals($response['body']['email'], $email);
$this->assertEquals($response['body']['name'], $name);
@ -92,7 +93,7 @@ class AccountConsoleClientTest extends Scope
$this->assertEquals(201, $response['headers']['status-code']);
$this->assertNotEmpty($response['body']);
$this->assertNotEmpty($response['body']['$id']);
$this->assertEquals(true, DateTime::isValid($response['body']['registration']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($response['body']['registration']));
$this->assertEquals($response['body']['email'], $email);
$this->assertEquals($response['body']['name'], $name);
}

18
tests/e2e/Services/Account/AccountCustomClientTest.php
View file

@ -477,8 +477,7 @@ class AccountCustomClientTest extends Scope
$this->assertIsArray($response['body']);
$this->assertNotEmpty($response['body']);
$this->assertNotEmpty($response['body']['$id']);
$dateValidator = new DatetimeValidator();
$this->assertEquals(true, $dateValidator->isValid($response['body']['registration']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($response['body']['registration']));
$this->assertEquals($response['body']['email'], $email);
$response = $this->client->call(Client::METHOD_POST, '/account/sessions/email', array_merge([
@ -729,8 +728,7 @@ class AccountCustomClientTest extends Scope
$this->assertEquals(201, $response['headers']['status-code']);
$this->assertNotEmpty($response['body']['$id']);
$this->assertEmpty($response['body']['secret']);
$dateValidator = new DatetimeValidator();
$this->assertEquals(true, $dateValidator->isValid($response['body']['expire']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($response['body']['expire']));
$userId = $response['body']['userId'];
@ -830,8 +828,7 @@ class AccountCustomClientTest extends Scope
$this->assertEquals($response['headers']['status-code'], 200);
$this->assertNotEmpty($response['body']);
$this->assertNotEmpty($response['body']['$id']);
$dateValidator = new DatetimeValidator();
$this->assertEquals(true, $dateValidator->isValid($response['body']['registration']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($response['body']['registration']));
$this->assertEquals($response['body']['phone'], $number);
$this->assertTrue($response['body']['phoneVerification']);
@ -882,8 +879,7 @@ class AccountCustomClientTest extends Scope
$this->assertIsArray($response['body']);
$this->assertNotEmpty($response['body']);
$this->assertNotEmpty($response['body']['$id']);
$dateValidator = new DatetimeValidator();
$this->assertEquals(true, $dateValidator->isValid($response['body']['registration']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($response['body']['registration']));
$this->assertEquals($response['body']['email'], $email);
$response = $this->client->call(Client::METHOD_POST, '/account/sessions/email', array_merge([
@ -925,8 +921,7 @@ class AccountCustomClientTest extends Scope
$this->assertIsArray($response['body']);
$this->assertNotEmpty($response['body']);
$this->assertNotEmpty($response['body']['$id']);
$dateValidator = new DatetimeValidator();
$this->assertEquals(true, $dateValidator->isValid($response['body']['registration']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($response['body']['registration']));
$this->assertEquals($response['body']['phone'], $newPhone);
/**
@ -976,8 +971,7 @@ class AccountCustomClientTest extends Scope
$this->assertEquals(201, $response['headers']['status-code']);
$this->assertNotEmpty($response['body']['$id']);
$this->assertEmpty($response['body']['secret']);
$dateValidator = new DatetimeValidator();
$this->assertEquals(true, $dateValidator->isValid($response['body']['expire']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($response['body']['expire']));
\sleep(2);

1
tests/e2e/Services/Databases/DatabasesBase.php
View file

@ -4,6 +4,7 @@ namespace Tests\E2E\Services\Databases;
use Tests\E2E\Client;
use Utopia\Database\Helpers\ID;
use Utopia\Database\DateTime;
use Utopia\Database\Helpers\Permission;
use Utopia\Database\Helpers\Role;
use Utopia\Database\Validator\DatetimeValidator;

73
tests/e2e/Services/Databases/DatabasesConsoleClientTest.php
View file

@ -56,47 +56,39 @@ class DatabasesConsoleClientTest extends Scope
/**
* @depends testCreateCollection
*/
// public function testGetDatabaseUsage(array $data)
// {
// $databaseId = $data['databaseId'];
// /**
// * Test for FAILURE
// */
public function testGetDatabaseUsage(array $data)
{
$databaseId = $data['databaseId'];
/**
* Test for FAILURE
*/
// $response = $this->client->call(Client::METHOD_GET, '/databases/' . $databaseId . '/usage', array_merge([
// 'content-type' => 'application/json',
// 'x-appwrite-project' => $this->getProject()['$id']
// ], $this->getHeaders()), [
// 'range' => '32h'
// ]);
$response = $this->client->call(Client::METHOD_GET, '/databases/' . $databaseId . '/usage', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id']
], $this->getHeaders()), [
'range' => '32h'
]);
// $this->assertEquals(400, $response['headers']['status-code']);
$this->assertEquals(400, $response['headers']['status-code']);
// /**
// * Test for SUCCESS
// */
/**
* Test for SUCCESS
*/
// $response = $this->client->call(Client::METHOD_GET, '/databases/' . $databaseId . '/usage', array_merge([
// 'content-type' => 'application/json',
// 'x-appwrite-project' => $this->getProject()['$id']
// ], $this->getHeaders()), [
// 'range' => '24h'
// ]);
$response = $this->client->call(Client::METHOD_GET, '/databases/' . $databaseId . '/usage', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id']
], $this->getHeaders()), [
'range' => '24h'
]);
// $this->assertEquals(200, $response['headers']['status-code']);
// $this->assertEquals(count($response['body']), 11);
// $this->assertEquals($response['body']['range'], '24h');
// $this->assertIsArray($response['body']['documentsCount']);
// $this->assertIsArray($response['body']['collectionsCount']);
// $this->assertIsArray($response['body']['documentsCreate']);
// $this->assertIsArray($response['body']['documentsRead']);
// $this->assertIsArray($response['body']['documentsUpdate']);
// $this->assertIsArray($response['body']['documentsDelete']);
// $this->assertIsArray($response['body']['collectionsCreate']);
// $this->assertIsArray($response['body']['collectionsRead']);
// $this->assertIsArray($response['body']['collectionsUpdate']);
// $this->assertIsArray($response['body']['collectionsDelete']);
// }
$this->assertEquals(200, $response['headers']['status-code']);
$this->assertEquals(count($response['body']), 3);
$this->assertEquals($response['body']['range'], '24h');
$this->assertIsArray($response['body']['documentsTotal']);
$this->assertIsArray($response['body']['collectionsTotal']);
}
/**
@ -136,15 +128,10 @@ class DatabasesConsoleClientTest extends Scope
], $this->getHeaders()), [
'range' => '24h'
]);
$this->assertEquals(200, $response['headers']['status-code']);
$this->assertEquals(count($response['body']), 6);
$this->assertEquals(count($response['body']), 2);
$this->assertEquals($response['body']['range'], '24h');
$this->assertIsArray($response['body']['documentsCount']);
$this->assertIsArray($response['body']['documentsCreate']);
$this->assertIsArray($response['body']['documentsRead']);
$this->assertIsArray($response['body']['documentsUpdate']);
$this->assertIsArray($response['body']['documentsDelete']);
$this->assertIsArray($response['body']['documentsTotal']);
}
/**

17
tests/e2e/Services/Functions/FunctionsConsoleClientTest.php
View file

@ -90,16 +90,15 @@ class FunctionsConsoleClientTest extends Scope
]);
$this->assertEquals($response['headers']['status-code'], 200);
$this->assertEquals(count($response['body']), 9);
$this->assertEquals(count($response['body']), 8);
$this->assertEquals($response['body']['range'], '24h');
$this->assertIsArray($response['body']['executionsTotal']);
$this->assertIsArray($response['body']['executionsFailure']);
$this->assertIsArray($response['body']['executionsSuccess']);
$this->assertIsArray($response['body']['executionsTime']);
$this->assertIsArray($response['body']['buildsTotal']);
$this->assertIsArray($response['body']['buildsFailure']);
$this->assertIsArray($response['body']['buildsSuccess']);
$this->assertIsArray($response['body']['buildsTime']);
$this->assertIsArray($response['body']['deployments']);
$this->assertIsArray($response['body']['deploymentsStorage']);
$this->assertIsArray($response['body']['builds']);
$this->assertIsArray($response['body']['buildsStorage']);
$this->assertIsArray($response['body']['buildsCompute']);
$this->assertIsArray($response['body']['executions']);
$this->assertIsArray($response['body']['executionsCompute']);
}
/**

2
tests/e2e/Services/Functions/FunctionsCustomClientTest.php
View file

@ -7,8 +7,6 @@ use Tests\E2E\Client;
use Tests\E2E\Scopes\ProjectCustom;
use Tests\E2E\Scopes\Scope;
use Tests\E2E\Scopes\SideClient;
use Utopia\CLI\Console;
use Utopia\Database\Database;
use Utopia\Database\Helpers\ID;
use Utopia\Database\Helpers\Role;

16
tests/e2e/Services/Functions/FunctionsCustomServerTest.php
View file

@ -8,6 +8,7 @@ use Tests\E2E\Client;
use Tests\E2E\Scopes\ProjectCustom;
use Tests\E2E\Scopes\Scope;
use Tests\E2E\Scopes\SideServer;
use Utopia\Database\DateTime;
use Utopia\Database\Helpers\ID;
use Utopia\Database\Validator\DatetimeValidator;
@ -369,8 +370,7 @@ class FunctionsCustomServerTest extends Scope
$this->assertEquals(202, $deployment['headers']['status-code']);
$this->assertNotEmpty($deployment['body']['$id']);
$dateValidator = new DatetimeValidator();
$this->assertEquals(true, $dateValidator->isValid($deployment['body']['$createdAt']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($deployment['body']['$createdAt']));
$this->assertEquals('index.php', $deployment['body']['entrypoint']);
// Wait for deployment to build.
@ -419,8 +419,7 @@ class FunctionsCustomServerTest extends Scope
$this->assertEquals(202, $largeTag['headers']['status-code']);
$this->assertNotEmpty($largeTag['body']['$id']);
$dateValidator = new DatetimeValidator();
$this->assertEquals(true, $dateValidator->isValid($largeTag['body']['$createdAt']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($largeTag['body']['$createdAt']));
$this->assertEquals('index.php', $largeTag['body']['entrypoint']);
$this->assertGreaterThan(10000, $largeTag['body']['size']);
@ -609,8 +608,7 @@ class FunctionsCustomServerTest extends Scope
$this->assertEquals(202, $execution['headers']['status-code']);
$this->assertNotEmpty($execution['body']['$id']);
$this->assertNotEmpty($execution['body']['functionId']);
$dateValidator = new DatetimeValidator();
$this->assertEquals(true, $dateValidator->isValid($execution['body']['$createdAt']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($execution['body']['$createdAt']));
$this->assertEquals($data['functionId'], $execution['body']['functionId']);
$this->assertEquals('waiting', $execution['body']['status']);
$this->assertEquals(0, $execution['body']['statusCode']);
@ -627,7 +625,7 @@ class FunctionsCustomServerTest extends Scope
$this->assertNotEmpty($execution['body']['$id']);
$this->assertNotEmpty($execution['body']['functionId']);
$this->assertEquals(true, $dateValidator->isValid($execution['body']['$createdAt']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($execution['body']['$createdAt']));
$this->assertEquals($data['functionId'], $execution['body']['functionId']);
$this->assertEquals('completed', $execution['body']['status']);
$this->assertEquals(200, $execution['body']['statusCode']);
@ -639,7 +637,7 @@ class FunctionsCustomServerTest extends Scope
$this->assertStringContainsString('8.0', $execution['body']['response']);
$this->assertStringContainsString('êä', $execution['body']['response']); // tests unknown utf-8 chars
$this->assertEquals('', $execution['body']['stderr']);
$this->assertLessThan(1.500, $execution['body']['duration']);
$this->assertLessThan(3, $execution['body']['duration']);
/**
* Test for FAILURE
@ -918,7 +916,7 @@ class FunctionsCustomServerTest extends Scope
$this->assertEquals($executions['body']['executions'][0]['status'], 'failed');
$this->assertEquals($executions['body']['executions'][0]['statusCode'], 500);
$this->assertGreaterThan(2, $executions['body']['executions'][0]['duration']);
$this->assertLessThan(4, $executions['body']['executions'][0]['duration']);
$this->assertLessThan(6, $executions['body']['executions'][0]['duration']);
$this->assertEquals($executions['body']['executions'][0]['response'], '');
$this->assertEquals($executions['body']['executions'][0]['stderr'], 'An internal curl error has occurred within the executor! Error Msg: Operation timed out');

16
tests/e2e/Services/Projects/ProjectsConsoleClientTest.php
View file

@ -6,9 +6,7 @@ use Appwrite\Auth\Auth;
use Tests\E2E\Scopes\Scope;
use Tests\E2E\Scopes\ProjectConsole;
use Tests\E2E\Scopes\SideClient;
use Tests\E2E\Services\Projects\ProjectsBase;
use Tests\E2E\Client;
use Utopia\Database\Database;
use Utopia\Database\DateTime;
use Utopia\Database\Helpers\ID;
@ -355,14 +353,14 @@ class ProjectsConsoleClientTest extends Scope
$this->assertEquals(count($response['body']), 9);
$this->assertNotEmpty($response['body']);
$this->assertEquals('30d', $response['body']['range']);
$this->assertIsArray($response['body']['requests']);
$this->assertIsArray($response['body']['requestsTotal']);
$this->assertIsArray($response['body']['network']);
$this->assertIsArray($response['body']['executions']);
$this->assertIsArray($response['body']['documents']);
$this->assertIsArray($response['body']['databases']);
$this->assertIsArray($response['body']['buckets']);
$this->assertIsArray($response['body']['users']);
$this->assertIsArray($response['body']['storage']);
$this->assertIsArray($response['body']['executionsTotal']);
$this->assertIsArray($response['body']['documentsTotal']);
$this->assertIsArray($response['body']['databasesTotal']);
$this->assertIsArray($response['body']['bucketsTotal']);
$this->assertIsArray($response['body']['usersTotal']);
$this->assertIsArray($response['body']['filesStorage']);
/**
* Test for FAILURE

2
tests/e2e/Services/Realtime/RealtimeCustomClientTest.php
View file

@ -1293,7 +1293,7 @@ class RealtimeCustomClientTest extends Scope
$this->assertNotEmpty($deployment['body']['$id']);
// Wait for deployment to be built.
sleep(5);
sleep(10);
$response = $this->client->call(Client::METHOD_PATCH, '/functions/' . $functionId . '/deployments/' . $deploymentId, array_merge([
'content-type' => 'application/json',

10
tests/e2e/Services/Storage/StorageBase.php
View file

@ -4,6 +4,7 @@ namespace Tests\E2E\Services\Storage;
use CURLFile;
use Tests\E2E\Client;
use Utopia\Database\DateTime;
use Utopia\Database\Helpers\ID;
use Utopia\Database\Helpers\Permission;
use Utopia\Database\Helpers\Role;
@ -284,8 +285,7 @@ trait StorageBase
]);
$this->assertEquals(201, $file['headers']['status-code']);
$this->assertNotEmpty($file['body']['$id']);
$dateValidator = new DatetimeValidator();
$this->assertEquals(true, $dateValidator->isValid($file['body']['$createdAt']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($file['body']['$createdAt']));
$this->assertEquals('logo.png', $file['body']['name']);
$this->assertEquals('image/png', $file['body']['mimeType']);
$this->assertEquals(47218, $file['body']['sizeOriginal']);
@ -375,8 +375,7 @@ trait StorageBase
$this->assertEquals(200, $file1['headers']['status-code']);
$this->assertNotEmpty($file1['body']['$id']);
$dateValidator = new DatetimeValidator();
$this->assertEquals(true, $dateValidator->isValid($file1['body']['$createdAt']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($file1['body']['$createdAt']));
$this->assertEquals('logo.png', $file1['body']['name']);
$this->assertEquals('image/png', $file1['body']['mimeType']);
$this->assertEquals(47218, $file1['body']['sizeOriginal']);
@ -715,8 +714,7 @@ trait StorageBase
$this->assertEquals(200, $file['headers']['status-code']);
$this->assertNotEmpty($file['body']['$id']);
$dateValidator = new DatetimeValidator();
$this->assertEquals(true, $dateValidator->isValid($file['body']['$createdAt']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($file['body']['$createdAt']));
$this->assertEquals('logo.png', $file['body']['name']);
$this->assertEquals('image/png', $file['body']['mimeType']);
$this->assertEquals(47218, $file['body']['sizeOriginal']);

15
tests/e2e/Services/Storage/StorageConsoleClientTest.php
View file

@ -39,10 +39,11 @@ class StorageConsoleClientTest extends Scope
]);
$this->assertEquals($response['headers']['status-code'], 200);
$this->assertEquals(12, count($response['body']));
$this->assertEquals(4, count($response['body']));
$this->assertEquals($response['body']['range'], '24h');
$this->assertIsArray($response['body']['storage']);
$this->assertIsArray($response['body']['filesCount']);
$this->assertIsArray($response['body']['bucketsTotal']);
$this->assertIsArray($response['body']['filesTotal']);
$this->assertIsArray($response['body']['filesStorage']);
}
public function testGetStorageBucketUsage()
@ -94,13 +95,9 @@ class StorageConsoleClientTest extends Scope
]);
$this->assertEquals($response['headers']['status-code'], 200);
$this->assertEquals(count($response['body']), 7);
$this->assertEquals(count($response['body']), 3);
$this->assertEquals($response['body']['range'], '24h');
$this->assertIsArray($response['body']['filesCount']);
$this->assertIsArray($response['body']['filesCreate']);
$this->assertIsArray($response['body']['filesRead']);
$this->assertIsArray($response['body']['filesUpdate']);
$this->assertIsArray($response['body']['filesDelete']);
$this->assertIsArray($response['body']['filesTotal']);
$this->assertIsArray($response['body']['filesStorage']);
}
}

31
tests/e2e/Services/Storage/StorageCustomClientTest.php
View file

@ -7,6 +7,7 @@ use Tests\E2E\Client;
use Tests\E2E\Scopes\Scope;
use Tests\E2E\Scopes\ProjectCustom;
use Tests\E2E\Scopes\SideClient;
use Utopia\Database\DateTime;
use Utopia\Database\Helpers\ID;
use Utopia\Database\Helpers\Permission;
use Utopia\Database\Helpers\Role;
@ -57,8 +58,7 @@ class StorageCustomClientTest extends Scope
$fileId = $file['body']['$id'];
$this->assertEquals($file['headers']['status-code'], 201);
$this->assertNotEmpty($fileId);
$dateValidator = new DatetimeValidator();
$this->assertEquals(true, $dateValidator->isValid($file['body']['$createdAt']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($file['body']['$createdAt']));
$this->assertEquals('permissions.png', $file['body']['name']);
$this->assertEquals('image/png', $file['body']['mimeType']);
$this->assertEquals(47218, $file['body']['sizeOriginal']);
@ -154,8 +154,7 @@ class StorageCustomClientTest extends Scope
$fileId = $file['body']['$id'];
$this->assertEquals($file['headers']['status-code'], 201);
$this->assertNotEmpty($fileId);
$dateValidator = new DatetimeValidator();
$this->assertEquals(true, $dateValidator->isValid($file['body']['$createdAt']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($file['body']['$createdAt']));
$this->assertEquals('permissions.png', $file['body']['name']);
$this->assertEquals('image/png', $file['body']['mimeType']);
$this->assertEquals(47218, $file['body']['sizeOriginal']);
@ -241,8 +240,7 @@ class StorageCustomClientTest extends Scope
$fileId = $file['body']['$id'];
$this->assertEquals($file['headers']['status-code'], 201);
$this->assertNotEmpty($fileId);
$dateValidator = new DatetimeValidator();
$this->assertEquals(true, $dateValidator->isValid($file['body']['$createdAt']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($file['body']['$createdAt']));
$this->assertEquals('permissions.png', $file['body']['name']);
$this->assertEquals('image/png', $file['body']['mimeType']);
$this->assertEquals(47218, $file['body']['sizeOriginal']);
@ -367,8 +365,7 @@ class StorageCustomClientTest extends Scope
$fileId = $file['body']['$id'];
$this->assertEquals($file['headers']['status-code'], 201);
$this->assertNotEmpty($fileId);
$dateValidator = new DatetimeValidator();
$this->assertEquals(true, $dateValidator->isValid($file['body']['$createdAt']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($file['body']['$createdAt']));
$this->assertEquals('permissions.png', $file['body']['name']);
$this->assertEquals('image/png', $file['body']['mimeType']);
$this->assertEquals(47218, $file['body']['sizeOriginal']);
@ -546,8 +543,7 @@ class StorageCustomClientTest extends Scope
$fileId = $file['body']['$id'];
$this->assertEquals($file['headers']['status-code'], 201);
$this->assertNotEmpty($fileId);
$dateValidator = new DatetimeValidator();
$this->assertEquals(true, $dateValidator->isValid($file['body']['$createdAt']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($file['body']['$createdAt']));
$this->assertEquals('permissions.png', $file['body']['name']);
$this->assertEquals('image/png', $file['body']['mimeType']);
$this->assertEquals(47218, $file['body']['sizeOriginal']);
@ -709,8 +705,7 @@ class StorageCustomClientTest extends Scope
$fileId = $file1['body']['$id'];
$this->assertEquals($file1['headers']['status-code'], 201);
$this->assertNotEmpty($fileId);
$dateValidator = new DatetimeValidator();
$this->assertEquals(true, $dateValidator->isValid($file1['body']['$createdAt']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($file1['body']['$createdAt']));
$this->assertEquals('permissions.png', $file1['body']['name']);
$this->assertEquals('image/png', $file1['body']['mimeType']);
$this->assertEquals(47218, $file1['body']['sizeOriginal']);
@ -799,8 +794,7 @@ class StorageCustomClientTest extends Scope
$fileId = $file1['body']['$id'];
$this->assertEquals($file1['headers']['status-code'], 201);
$this->assertNotEmpty($fileId);
$dateValidator = new DatetimeValidator();
$this->assertEquals(true, $dateValidator->isValid($file1['body']['$createdAt']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($file1['body']['$createdAt']));
$this->assertEquals('permissions.png', $file1['body']['name']);
$this->assertEquals('image/png', $file1['body']['mimeType']);
$this->assertEquals(47218, $file1['body']['sizeOriginal']);
@ -889,8 +883,7 @@ class StorageCustomClientTest extends Scope
$fileId = $file1['body']['$id'];
$this->assertEquals($file1['headers']['status-code'], 201);
$this->assertNotEmpty($fileId);
$dateValidator = new DatetimeValidator();
$this->assertEquals(true, $dateValidator->isValid($file1['body']['$createdAt']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($file1['body']['$createdAt']));
$this->assertEquals('permissions.png', $file1['body']['name']);
$this->assertEquals('image/png', $file1['body']['mimeType']);
$this->assertEquals(47218, $file1['body']['sizeOriginal']);
@ -1029,8 +1022,7 @@ class StorageCustomClientTest extends Scope
$fileId = $file['body']['$id'];
$this->assertEquals($file['headers']['status-code'], 201);
$this->assertNotEmpty($fileId);
$dateValidator = new DatetimeValidator();
$this->assertEquals(true, $dateValidator->isValid($file['body']['$createdAt']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($file['body']['$createdAt']));
$this->assertEquals('permissions.png', $file['body']['name']);
$this->assertEquals('image/png', $file['body']['mimeType']);
$this->assertEquals(47218, $file['body']['sizeOriginal']);
@ -1265,8 +1257,7 @@ class StorageCustomClientTest extends Scope
$this->assertContains(Permission::read(Role::user($this->getUser()['$id'])), $file['body']['$permissions']);
$this->assertContains(Permission::update(Role::user($this->getUser()['$id'])), $file['body']['$permissions']);
$this->assertContains(Permission::delete(Role::user($this->getUser()['$id'])), $file['body']['$permissions']);
$dateValidator = new DatetimeValidator();
$this->assertEquals(true, $dateValidator->isValid($file['body']['$createdAt']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($file['body']['$createdAt']));
$this->assertEquals('permissions.png', $file['body']['name']);
$this->assertEquals('image/png', $file['body']['mimeType']);
$this->assertEquals(47218, $file['body']['sizeOriginal']);

6
tests/e2e/Services/Storage/StorageCustomServerTest.php
View file

@ -30,8 +30,7 @@ class StorageCustomServerTest extends Scope
]);
$this->assertEquals(201, $bucket['headers']['status-code']);
$this->assertNotEmpty($bucket['body']['$id']);
$dateValidator = new DatetimeValidator();
$this->assertEquals(true, $dateValidator->isValid($bucket['body']['$createdAt']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($bucket['body']['$createdAt']));
$this->assertIsArray($bucket['body']['$permissions']);
$this->assertIsArray($bucket['body']['allowedFileExtensions']);
$this->assertEquals('Test Bucket', $bucket['body']['name']);
@ -229,8 +228,7 @@ class StorageCustomServerTest extends Scope
]);
$this->assertEquals(200, $bucket['headers']['status-code']);
$this->assertNotEmpty($bucket['body']['$id']);
$dateValidator = new DatetimeValidator();
$this->assertEquals(true, $dateValidator->isValid($bucket['body']['$createdAt']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($bucket['body']['$createdAt']));
$this->assertIsArray($bucket['body']['$permissions']);
$this->assertIsArray($bucket['body']['allowedFileExtensions']);
$this->assertEquals('Test Bucket Updated', $bucket['body']['name']);

19
tests/e2e/Services/Teams/TeamsBase.php
View file

@ -3,6 +3,7 @@
namespace Tests\E2E\Services\Teams;
use Tests\E2E\Client;
use Utopia\Database\DateTime;
use Utopia\Database\Helpers\ID;
use Utopia\Database\Validator\DatetimeValidator;
@ -27,8 +28,7 @@ trait TeamsBase
$this->assertEquals('Arsenal', $response1['body']['name']);
$this->assertGreaterThan(-1, $response1['body']['total']);
$this->assertIsInt($response1['body']['total']);
$dateValidator = new DatetimeValidator();
$this->assertEquals(true, $dateValidator->isValid($response1['body']['$createdAt']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($response1['body']['$createdAt']));
$teamUid = $response1['body']['$id'];
$teamName = $response1['body']['name'];
@ -48,7 +48,7 @@ trait TeamsBase
$this->assertEquals('Manchester United', $response2['body']['name']);
$this->assertGreaterThan(-1, $response2['body']['total']);
$this->assertIsInt($response2['body']['total']);
$this->assertEquals(true, $dateValidator->isValid($response2['body']['$createdAt']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($response2['body']['$createdAt']));
$response3 = $this->client->call(Client::METHOD_POST, '/teams', array_merge([
'content-type' => 'application/json',
@ -63,7 +63,7 @@ trait TeamsBase
$this->assertEquals('Newcastle', $response3['body']['name']);
$this->assertGreaterThan(-1, $response3['body']['total']);
$this->assertIsInt($response3['body']['total']);
$this->assertEquals(true, $dateValidator->isValid($response3['body']['$createdAt']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($response3['body']['$createdAt']));
/**
* Test for FAILURE
*/
@ -98,8 +98,7 @@ trait TeamsBase
$this->assertEquals('Arsenal', $response['body']['name']);
$this->assertGreaterThan(-1, $response['body']['total']);
$this->assertIsInt($response['body']['total']);
$dateValidator = new DatetimeValidator();
$this->assertEquals(true, $dateValidator->isValid($response['body']['$createdAt']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($response['body']['$createdAt']));
/**
* Test for FAILURE
@ -276,8 +275,7 @@ trait TeamsBase
$this->assertEquals('Demo', $response['body']['name']);
$this->assertGreaterThan(-1, $response['body']['total']);
$this->assertIsInt($response['body']['total']);
$dateValidator = new DatetimeValidator();
$this->assertEquals(true, $dateValidator->isValid($response['body']['$createdAt']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($response['body']['$createdAt']));
$response = $this->client->call(Client::METHOD_PUT, '/teams/' . $response['body']['$id'], array_merge([
'content-type' => 'application/json',
@ -292,7 +290,7 @@ trait TeamsBase
$this->assertEquals('Demo New', $response['body']['name']);
$this->assertGreaterThan(-1, $response['body']['total']);
$this->assertIsInt($response['body']['total']);
$this->assertEquals(true, $dateValidator->isValid($response['body']['$createdAt']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($response['body']['$createdAt']));
/**
* Test for FAILURE
@ -328,8 +326,7 @@ trait TeamsBase
$this->assertEquals('Demo', $response['body']['name']);
$this->assertGreaterThan(-1, $response['body']['total']);
$this->assertIsInt($response['body']['total']);
$dateValidator = new DatetimeValidator();
$this->assertEquals(true, $dateValidator->isValid($response['body']['$createdAt']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($response['body']['$createdAt']));
$response = $this->client->call(Client::METHOD_DELETE, '/teams/' . $teamUid, array_merge([
'content-type' => 'application/json',

14
tests/e2e/Services/Teams/TeamsBaseClient.php
View file

@ -3,6 +3,7 @@
namespace Tests\E2E\Services\Teams;
use Tests\E2E\Client;
use Utopia\Database\DateTime;
use Utopia\Database\Helpers\ID;
use Utopia\Database\Validator\DatetimeValidator;
@ -150,8 +151,7 @@ trait TeamsBaseClient
$this->assertNotEmpty($response['body']['teamId']);
$this->assertNotEmpty($response['body']['teamName']);
$this->assertCount(2, $response['body']['roles']);
$dateValidator = new DatetimeValidator();
$this->assertEquals(false, $dateValidator->isValid($response['body']['joined'])); // is null in DB
$this->assertEquals(false, (new DatetimeValidator())->isValid($response['body']['joined'])); // is null in DB
$this->assertEquals(false, $response['body']['confirm']);
/**
@ -204,8 +204,7 @@ trait TeamsBaseClient
$this->assertNotEmpty($response['body']['teamId']);
$this->assertNotEmpty($response['body']['teamName']);
$this->assertCount(2, $response['body']['roles']);
$dateValidator = new DatetimeValidator();
$this->assertEquals(false, $dateValidator->isValid($response['body']['joined'])); // is null in DB
$this->assertEquals(false, (new DatetimeValidator())->isValid($response['body']['joined'])); // is null in DB
$this->assertEquals(false, $response['body']['confirm']);
$lastEmail = $this->getLastEmail();
@ -339,8 +338,7 @@ trait TeamsBaseClient
$this->assertNotEmpty($response['body']['userId']);
$this->assertNotEmpty($response['body']['teamId']);
$this->assertCount(2, $response['body']['roles']);
$dateValidator = new DatetimeValidator();
$this->assertEquals(true, $dateValidator->isValid($response['body']['joined']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($response['body']['joined']));
$this->assertEquals(true, $response['body']['confirm']);
$session = $this->client->parseCookie((string)$response['headers']['set-cookie'])['a_session_' . $this->getProject()['$id']];
$data['session'] = $session;
@ -372,7 +370,7 @@ trait TeamsBaseClient
$this->assertIsArray($response['body']);
$this->assertNotEmpty($response['body']);
$this->assertNotEmpty($response['body']['$id']);
$this->assertEquals(true, $dateValidator->isValid($response['body']['registration']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($response['body']['registration']));
$this->assertEquals($response['body']['email'], $email);
$this->assertEquals($response['body']['name'], $name);
@ -406,7 +404,7 @@ trait TeamsBaseClient
$this->assertIsArray($response['body']);
$this->assertNotEmpty($response['body']);
$this->assertNotEmpty($response['body']['$id']);
$this->assertEquals(true, $dateValidator->isValid($response['body']['registration']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($response['body']['registration']));
$this->assertEquals($response['body']['email'], $email);
$this->assertEquals($response['body']['name'], $name);

13
tests/e2e/Services/Teams/TeamsBaseServer.php
View file

@ -3,6 +3,8 @@
namespace Tests\E2E\Services\Teams;
use Tests\E2E\Client;
use Utopia\Database\Database;
use Utopia\Database\DateTime;
use Utopia\Database\Validator\DatetimeValidator;
trait TeamsBaseServer
@ -56,8 +58,7 @@ trait TeamsBaseServer
$this->assertNotEmpty($response['body']['teamId']);
$this->assertNotEmpty($response['body']['teamName']);
$this->assertCount(2, $response['body']['roles']);
$dateValidator = new DatetimeValidator();
$this->assertEquals(true, $dateValidator->isValid($response['body']['joined'])); // is null in DB
$this->assertEquals(true, (new DatetimeValidator())->isValid($response['body']['joined'])); // is null in DB
$this->assertEquals(true, $response['body']['confirm']);
/**
@ -108,8 +109,7 @@ trait TeamsBaseServer
$this->assertEquals($email, $response['body']['userEmail']);
$this->assertNotEmpty($response['body']['teamId']);
$this->assertCount(2, $response['body']['roles']);
$dateValidator = new DatetimeValidator();
$this->assertEquals(true, $dateValidator->isValid($response['body']['joined']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($response['body']['joined']));
$this->assertEquals(true, $response['body']['confirm']);
$userUid = $response['body']['userId'];
@ -252,8 +252,7 @@ trait TeamsBaseServer
$this->assertEquals('Arsenal', $response['body']['name']);
$this->assertEquals(1, $response['body']['total']);
$this->assertIsInt($response['body']['total']);
$dateValidator = new DatetimeValidator();
$this->assertEquals(true, $dateValidator->isValid($response['body']['$createdAt']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($response['body']['$createdAt']));
/** Delete User */
$user = $this->client->call(Client::METHOD_DELETE, '/users/' . $userUid, array_merge([
@ -278,6 +277,6 @@ trait TeamsBaseServer
$this->assertEquals('Arsenal', $response['body']['name']);
$this->assertEquals(0, $response['body']['total']);
$this->assertIsInt($response['body']['total']);
$this->assertEquals(true, $dateValidator->isValid($response['body']['$createdAt']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($response['body']['$createdAt']));
}
}

43
tests/e2e/Services/Users/UsersConsoleClientTest.php
View file

@ -23,17 +23,6 @@ class UsersConsoleClientTest extends Scope
'x-appwrite-project' => $this->getProject()['$id']
], $this->getHeaders()), [
'range' => '32h',
'provider' => 'email'
]);
$this->assertEquals($response['headers']['status-code'], 400);
$response = $this->client->call(Client::METHOD_GET, '/users/usage', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id']
], $this->getHeaders()), [
'range' => '24h',
'provider' => 'some-random-provider'
]);
$this->assertEquals($response['headers']['status-code'], 400);
@ -46,38 +35,12 @@ class UsersConsoleClientTest extends Scope
'x-appwrite-project' => $this->getProject()['$id']
], $this->getHeaders()), [
'range' => '24h',
'provider' => 'email'
]);
$this->assertEquals($response['headers']['status-code'], 200);
$this->assertEquals(count($response['body']), 9);
$this->assertEquals(count($response['body']), 3);
$this->assertEquals($response['body']['range'], '24h');
$this->assertIsArray($response['body']['usersCount']);
$this->assertIsArray($response['body']['usersCreate']);
$this->assertIsArray($response['body']['usersRead']);
$this->assertIsArray($response['body']['usersUpdate']);
$this->assertIsArray($response['body']['usersDelete']);
$this->assertIsArray($response['body']['sessionsCreate']);
$this->assertIsArray($response['body']['sessionsProviderCreate']);
$this->assertIsArray($response['body']['sessionsDelete']);
$response = $this->client->call(Client::METHOD_GET, '/users/usage', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id']
], $this->getHeaders()), [
'range' => '24h'
]);
$this->assertEquals($response['headers']['status-code'], 200);
$this->assertEquals(count($response['body']), 9);
$this->assertEquals($response['body']['range'], '24h');
$this->assertIsArray($response['body']['usersCount']);
$this->assertIsArray($response['body']['usersCreate']);
$this->assertIsArray($response['body']['usersRead']);
$this->assertIsArray($response['body']['usersUpdate']);
$this->assertIsArray($response['body']['usersDelete']);
$this->assertIsArray($response['body']['sessionsCreate']);
$this->assertIsArray($response['body']['sessionsProviderCreate']);
$this->assertIsArray($response['body']['sessionsDelete']);
$this->assertIsArray($response['body']['usersTotal']);
$this->assertIsArray($response['body']['sessionsTotal']);
}
}

25
tests/e2e/Services/Webhooks/WebhooksBase.php
View file

@ -5,6 +5,7 @@ namespace Tests\E2E\Services\Webhooks;
use Appwrite\Tests\Retry;
use CURLFile;
use Tests\E2E\Client;
use Utopia\Database\DateTime;
use Utopia\Database\Helpers\ID;
use Utopia\Database\Helpers\Permission;
use Utopia\Database\Helpers\Role;
@ -529,8 +530,7 @@ trait WebhooksBase
$this->assertNotEmpty($webhook['data']['$id']);
$this->assertIsArray($webhook['data']['$permissions']);
$this->assertEquals($webhook['data']['name'], 'logo.png');
$dateValidator = new DatetimeValidator();
$this->assertEquals(true, $dateValidator->isValid($webhook['data']['$createdAt']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($webhook['data']['$createdAt']));
$this->assertNotEmpty($webhook['data']['signature']);
$this->assertEquals($webhook['data']['mimeType'], 'image/png');
$this->assertEquals($webhook['data']['sizeOriginal'], 47218);
@ -588,8 +588,7 @@ trait WebhooksBase
$this->assertNotEmpty($webhook['data']['$id']);
$this->assertIsArray($webhook['data']['$permissions']);
$this->assertEquals($webhook['data']['name'], 'logo.png');
$dateValidator = new DatetimeValidator();
$this->assertEquals(true, $dateValidator->isValid($webhook['data']['$createdAt']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($webhook['data']['$createdAt']));
$this->assertNotEmpty($webhook['data']['signature']);
$this->assertEquals($webhook['data']['mimeType'], 'image/png');
$this->assertEquals($webhook['data']['sizeOriginal'], 47218);
@ -639,8 +638,7 @@ trait WebhooksBase
$this->assertNotEmpty($webhook['data']['$id']);
$this->assertIsArray($webhook['data']['$permissions']);
$this->assertEquals($webhook['data']['name'], 'logo.png');
$dateValidator = new DatetimeValidator();
$this->assertEquals(true, $dateValidator->isValid($webhook['data']['$createdAt']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($webhook['data']['$createdAt']));
$this->assertNotEmpty($webhook['data']['signature']);
$this->assertEquals($webhook['data']['mimeType'], 'image/png');
$this->assertEquals($webhook['data']['sizeOriginal'], 47218);
@ -722,8 +720,7 @@ trait WebhooksBase
$this->assertEquals('Arsenal', $webhook['data']['name']);
$this->assertGreaterThan(-1, $webhook['data']['total']);
$this->assertIsInt($webhook['data']['total']);
$dateValidator = new DatetimeValidator();
$this->assertEquals(true, $dateValidator->isValid($webhook['data']['$createdAt']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($webhook['data']['$createdAt']));
/**
* Test for FAILURE
@ -768,8 +765,7 @@ trait WebhooksBase
$this->assertEquals('Demo New', $webhook['data']['name']);
$this->assertGreaterThan(-1, $webhook['data']['total']);
$this->assertIsInt($webhook['data']['total']);
$dateValidator = new DatetimeValidator();
$this->assertEquals(true, $dateValidator->isValid($webhook['data']['$createdAt']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($webhook['data']['$createdAt']));
/**
* Test for FAILURE
@ -818,8 +814,7 @@ trait WebhooksBase
$this->assertEquals('Chelsea', $webhook['data']['name']);
$this->assertGreaterThan(-1, $webhook['data']['total']);
$this->assertIsInt($webhook['data']['total']);
$dateValidator = new DatetimeValidator();
$this->assertEquals(true, $dateValidator->isValid($webhook['data']['$createdAt']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($webhook['data']['$createdAt']));
/**
* Test for FAILURE
@ -880,8 +875,7 @@ trait WebhooksBase
$this->assertNotEmpty($webhook['data']['userId']);
$this->assertNotEmpty($webhook['data']['teamId']);
$this->assertCount(2, $webhook['data']['roles']);
$dateValidator = new DatetimeValidator();
$this->assertEquals(true, $dateValidator->isValid($webhook['data']['invited']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($webhook['data']['invited']));
$this->assertEquals(('server' === $this->getSide()), $webhook['data']['confirm']);
/**
@ -953,8 +947,7 @@ trait WebhooksBase
$this->assertNotEmpty($webhook['data']['userId']);
$this->assertNotEmpty($webhook['data']['teamId']);
$this->assertCount(2, $webhook['data']['roles']);
$dateValidator = new DatetimeValidator();
$this->assertEquals(true, $dateValidator->isValid($webhook['data']['invited']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($webhook['data']['invited']));
$this->assertEquals(('server' === $this->getSide()), $webhook['data']['confirm']);
}
}

28
tests/e2e/Services/Webhooks/WebhooksCustomClientTest.php
View file

@ -7,6 +7,7 @@ use Tests\E2E\Client;
use Tests\E2E\Scopes\Scope;
use Tests\E2E\Scopes\ProjectCustom;
use Tests\E2E\Scopes\SideClient;
use Utopia\Database\DateTime;
use Utopia\Database\Helpers\ID;
use Utopia\Database\Validator\DatetimeValidator;
@ -60,8 +61,7 @@ class WebhooksCustomClientTest extends Scope
$this->assertEquals(empty($webhook['headers']['X-Appwrite-Webhook-User-Id']), true);
$this->assertNotEmpty($webhook['data']['$id']);
$this->assertEquals($webhook['data']['name'], $name);
$dateValidator = new DatetimeValidator();
$this->assertEquals(true, $dateValidator->isValid($webhook['data']['registration']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($webhook['data']['registration']));
$this->assertEquals($webhook['data']['status'], true);
$this->assertEquals($webhook['data']['email'], $email);
$this->assertEquals($webhook['data']['emailVerification'], false);
@ -137,8 +137,7 @@ class WebhooksCustomClientTest extends Scope
$this->assertEquals(empty($webhook['headers']['X-Appwrite-Webhook-User-Id'] ?? ''), ('server' === $this->getSide()));
$this->assertNotEmpty($webhook['data']['$id']);
$this->assertEquals($webhook['data']['name'], $name);
$dateValidator = new DatetimeValidator();
$this->assertEquals(true, $dateValidator->isValid($webhook['data']['registration']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($webhook['data']['registration']));
$this->assertEquals($webhook['data']['status'], false);
$this->assertEquals($webhook['data']['email'], $email);
$this->assertEquals($webhook['data']['emailVerification'], false);
@ -198,8 +197,7 @@ class WebhooksCustomClientTest extends Scope
$this->assertEquals(empty($webhook['headers']['X-Appwrite-Webhook-User-Id']), true);
$this->assertNotEmpty($webhook['data']['$id']);
$this->assertNotEmpty($webhook['data']['userId']);
$dateValidator = new DatetimeValidator();
$this->assertEquals(true, $dateValidator->isValid($webhook['data']['expire']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($webhook['data']['expire']));
$this->assertEquals($webhook['data']['ip'], '127.0.0.1');
$this->assertNotEmpty($webhook['data']['osCode']);
$this->assertIsString($webhook['data']['osCode']);
@ -374,8 +372,7 @@ class WebhooksCustomClientTest extends Scope
$this->assertEquals(empty($webhook['headers']['X-Appwrite-Webhook-User-Id'] ?? ''), ('server' === $this->getSide()));
$this->assertNotEmpty($webhook['data']['$id']);
$this->assertNotEmpty($webhook['data']['userId']);
$dateValidator = new DatetimeValidator();
$this->assertEquals(true, $dateValidator->isValid($webhook['data']['expire']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($webhook['data']['expire']));
$this->assertEquals($webhook['data']['ip'], '127.0.0.1');
$this->assertNotEmpty($webhook['data']['osCode']);
$this->assertIsString($webhook['data']['osCode']);
@ -687,8 +684,7 @@ class WebhooksCustomClientTest extends Scope
$this->assertNotEmpty($webhook['data']['$id']);
$this->assertNotEmpty($webhook['data']['userId']);
$this->assertNotEmpty($webhook['data']['secret']);
$dateValidator = new DatetimeValidator();
$this->assertEquals(true, $dateValidator->isValid($webhook['data']['expire']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($webhook['data']['expire']));
$data['secret'] = $webhook['data']['secret'];
@ -748,8 +744,7 @@ class WebhooksCustomClientTest extends Scope
$this->assertNotEmpty($webhook['data']['$id']);
$this->assertNotEmpty($webhook['data']['userId']);
$this->assertNotEmpty($webhook['data']['secret']);
$dateValidator = new DatetimeValidator();
$this->assertEquals(true, $dateValidator->isValid($webhook['data']['expire']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($webhook['data']['expire']));
$data['secret'] = $webhook['data']['secret'];
@ -805,8 +800,7 @@ class WebhooksCustomClientTest extends Scope
$this->assertNotEmpty($webhook['data']['$id']);
$this->assertNotEmpty($webhook['data']['userId']);
$this->assertNotEmpty($webhook['data']['secret']);
$dateValidator = new DatetimeValidator();
$this->assertEquals(true, $dateValidator->isValid($webhook['data']['expire']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($webhook['data']['expire']));
$data['secret'] = $webhook['data']['secret'];
@ -864,8 +858,7 @@ class WebhooksCustomClientTest extends Scope
$this->assertNotEmpty($webhook['data']['$id']);
$this->assertNotEmpty($webhook['data']['userId']);
$this->assertNotEmpty($webhook['data']['secret']);
$dateValidator = new DatetimeValidator();
$this->assertEquals(true, $dateValidator->isValid($webhook['data']['expire']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($webhook['data']['expire']));
$data['secret'] = $webhook['data']['secret'];
@ -928,8 +921,7 @@ class WebhooksCustomClientTest extends Scope
$this->assertNotEmpty($webhook['data']['userId']);
$this->assertNotEmpty($webhook['data']['teamId']);
$this->assertCount(2, $webhook['data']['roles']);
$dateValidator = new DatetimeValidator();
$this->assertEquals(true, $dateValidator->isValid($webhook['data']['joined']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($webhook['data']['joined']));
$this->assertEquals(true, $webhook['data']['confirm']);
/**

10
tests/e2e/Services/Webhooks/WebhooksCustomServerTest.php
View file

@ -8,6 +8,7 @@ use Tests\E2E\Scopes\ProjectCustom;
use Tests\E2E\Scopes\Scope;
use Tests\E2E\Scopes\SideServer;
use Utopia\CLI\Console;
use Utopia\Database\DateTime;
use Utopia\Database\Helpers\ID;
use Utopia\Database\Helpers\Permission;
use Utopia\Database\Helpers\Role;
@ -245,8 +246,7 @@ class WebhooksCustomServerTest extends Scope
$this->assertEquals(empty($webhook['headers']['X-Appwrite-Webhook-User-Id'] ?? ''), ('server' === $this->getSide()));
$this->assertNotEmpty($webhook['data']['$id']);
$this->assertEquals($webhook['data']['name'], $name);
$dateValidator = new DatetimeValidator();
$this->assertEquals(true, $dateValidator->isValid($webhook['data']['registration']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($webhook['data']['registration']));
$this->assertEquals($webhook['data']['status'], true);
$this->assertEquals($webhook['data']['email'], $email);
$this->assertEquals($webhook['data']['emailVerification'], false);
@ -337,8 +337,7 @@ class WebhooksCustomServerTest extends Scope
$this->assertEquals(empty($webhook['headers']['X-Appwrite-Webhook-User-Id'] ?? ''), ('server' === $this->getSide()));
$this->assertNotEmpty($webhook['data']['$id']);
$this->assertEquals($webhook['data']['name'], $data['name']);
$dateValidator = new DatetimeValidator();
$this->assertEquals(true, $dateValidator->isValid($webhook['data']['registration']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($webhook['data']['registration']));
$this->assertEquals($webhook['data']['status'], false);
$this->assertEquals($webhook['data']['email'], $data['email']);
$this->assertEquals($webhook['data']['emailVerification'], false);
@ -380,8 +379,7 @@ class WebhooksCustomServerTest extends Scope
$this->assertEquals(empty($webhook['headers']['X-Appwrite-Webhook-User-Id'] ?? ''), ('server' === $this->getSide()));
$this->assertNotEmpty($webhook['data']['$id']);
$this->assertEquals($webhook['data']['name'], $data['name']);
$dateValidator = new DatetimeValidator();
$this->assertEquals(true, $dateValidator->isValid($webhook['data']['registration']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($webhook['data']['registration']));
$this->assertEquals($webhook['data']['status'], false);
$this->assertEquals($webhook['data']['email'], $data['email']);
$this->assertEquals($webhook['data']['emailVerification'], false);

53
tests/resources/docker/docker-compose.yml
View file

@ -36,7 +36,7 @@ services:
- TESTING=true
- VERSION=dev
restart: unless-stopped
ports:
ports:
- 9501:80
networks:
- appwrite
@ -63,7 +63,6 @@ services:
- mariadb
- redis
# - clamav
- influxdb
environment:
- _APP_ENV
- _APP_OPTIONS_ABUSE
@ -79,8 +78,6 @@ services:
- _APP_DB_USER
- _APP_DB_PASS
- _APP_USAGE_STATS
- _APP_INFLUXDB_HOST
- _APP_INFLUXDB_PORT
- _APP_STORAGE_ANTIVIRUS=disabled
- _APP_STORAGE_LIMIT
- _APP_FUNCTIONS_TIMEOUT
@ -100,13 +97,10 @@ services:
- appwrite
depends_on:
- redis
- telegraf
environment:
- _APP_ENV
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_STATSD_HOST
- _APP_STATSD_PORT
appwrite-worker-audits:
entrypoint: worker-audits
@ -182,7 +176,7 @@ services:
depends_on:
- redis
- mariadb
volumes:
volumes:
- appwrite-uploads:/storage/uploads:rw
- appwrite-cache:/storage/cache:rw
environment:
@ -207,7 +201,7 @@ services:
depends_on:
- redis
- mariadb
volumes:
volumes:
- appwrite-config:/storage/config:rw
- appwrite-certificates:/storage/certificates:rw
environment:
@ -269,7 +263,7 @@ services:
- _APP_REDIS_PORT
- _APP_SMTP_HOST
- _APP_SMTP_PORT
appwrite-worker-builds:
entrypoint: worker-builds
container_name: appwrite-worker-builds
@ -277,7 +271,7 @@ services:
context: .
networks:
- appwrite
volumes:
volumes:
- ./app:/usr/src/code/app
- ./src:/usr/src/code/src
depends_on:
@ -368,21 +362,6 @@ services:
# volumes:
# - appwrite-uploads:/storage/uploads
influxdb:
image: influxdb:1.6
container_name: appwrite-influxdb
restart: unless-stopped
networks:
- appwrite
volumes:
- appwrite-influxdb:/var/lib/influxdb:rw
telegraf:
image: appwrite/telegraf:1.1.0
container_name: appwrite-telegraf
restart: unless-stopped
networks:
- appwrite
# redis-commander:
# image: rediscommander/redis-commander:latest
@ -407,24 +386,6 @@ services:
# - RESQUE_WEB_HTTP_BASIC_AUTH_USER=user
# - RESQUE_WEB_HTTP_BASIC_AUTH_PASSWORD=password
# chronograf:
# image: chronograf:1.5
# container_name: appwrite-chronograf
# restart: unless-stopped
# networks:
# - appwrite
# volumes:
# - appwrite-chronograf:/var/lib/chronograf
# ports:
# - "8888:8888"
# environment:
# - INFLUXDB_URL=http://influxdb:8086
# - KAPACITOR_URL=http://kapacitor:9092
# - AUTH_DURATION=48h
# - TOKEN_SECRET=duperduper5674829!jwt
# - GH_CLIENT_ID=d86f7145a41eacfc52cc
# - GH_CLIENT_SECRET=9e0081062367a2134e7f2ea95ba1a32d08b6c8ab
# - GH_ORGS=appwrite
# webgrind:
# image: 'jokkedk/webgrind:latest'
@ -444,6 +405,4 @@ volumes:
appwrite-uploads:
appwrite-certificates:
appwrite-functions:
appwrite-influxdb:
appwrite-chronograf:
appwrite-config:
appwrite-config:

1
tests/unit/Auth/AuthTest.php
View file

@ -9,7 +9,6 @@ use Utopia\Database\Helpers\ID;
use Utopia\Database\Helpers\Role;
use Utopia\Database\Validator\Authorization;
use PHPUnit\Framework\TestCase;
use Utopia\Database\Database;
use Utopia\Database\Validator\Roles;
class AuthTest extends TestCase

5
tests/unit/Docker/ComposeTest.php
View file

@ -28,8 +28,7 @@ class ComposeTest extends TestCase
public function testServices(): void
{
$this->assertCount(17, $this->object->getServices());
$this->assertEquals('appwrite-telegraf', $this->object->getService('telegraf')->getContainerName());
$this->assertCount(15, $this->object->getServices());
$this->assertEquals('appwrite', $this->object->getService('appwrite')->getContainerName());
$this->assertEquals('', $this->object->getService('appwrite')->getImageVersion());
$this->assertEquals('2.2', $this->object->getService('traefik')->getImageVersion());
@ -43,7 +42,7 @@ class ComposeTest extends TestCase
public function testVolumes(): void
{
$this->assertCount(9, $this->object->getVolumes());
$this->assertCount(7, $this->object->getVolumes());
$this->assertEquals('appwrite-mariadb', $this->object->getVolumes()[0]);
$this->assertEquals('appwrite-redis', $this->object->getVolumes()[1]);
$this->assertEquals('appwrite-cache', $this->object->getVolumes()[2]);

79
tests/unit/Usage/StatsTest.php
View file

@ -2,70 +2,53 @@
namespace Tests\Unit\Usage;
use Appwrite\Usage\Stats;
use Appwrite\URL\URL as AppwriteURL;
use PHPUnit\Framework\TestCase;
use Utopia\App;
use Utopia\DSN\DSN;
use Utopia\Queue;
use Utopia\Queue\Client;
use Utopia\Queue\Connection;
class StatsTest extends TestCase
{
/**
* @var Stats
*/
protected $object = null;
protected ?Connection $connection = null;
protected ?Client $client = null;
protected const QUEUE_NAME = 'usage-test-q';
public function setUp(): void
{
$host = App::getEnv('_APP_STATSD_HOST', 'telegraf');
$port = App::getEnv('_APP_STATSD_PORT', 8125);
$env = App::getEnv('_APP_CONNECTIONS_QUEUE', AppwriteURL::unparse([
'scheme' => 'redis',
'host' => App::getEnv('_APP_REDIS_HOST', 'redis'),
'port' => App::getEnv('_APP_REDIS_PORT', '6379'),
'user' => App::getEnv('_APP_REDIS_USER', ''),
'pass' => App::getEnv('_APP_REDIS_PASS', ''),
]));
$connection = new \Domnikl\Statsd\Connection\UdpSocket($host, $port);
$statsd = new \Domnikl\Statsd\Client($connection);
$this->object = new Stats($statsd);
$dsn = explode('=', $env);
$dsn = $dsn[1] ?? '';
$dsn = new DSN($dsn);
$this->connection = new Queue\Connection\Redis($dsn->getHost(), $dsn->getPort());
$this->client = new Client(self::QUEUE_NAME, $this->connection);
}
public function tearDown(): void
{
}
public function testNamespace(): void
public function testSamePayload(): void
{
$this->object->setNamespace('appwritetest.usage');
$this->assertEquals('appwritetest.usage', $this->object->getNamespace());
}
$inToQueue = [
'key_1' => 'value_1',
'key_2' => 'value_2',
];
public function testParams(): void
{
$this->object
->setParam('projectId', 'appwrite_test')
->setParam('projectInternalId', 1)
->setParam('networkRequestSize', 100)
;
$this->assertEquals('appwrite_test', $this->object->getParam('projectId'));
$this->assertEquals(1, $this->object->getParam('projectInternalId'));
$this->assertEquals(100, $this->object->getParam('networkRequestSize'));
$this->object->submit();
$this->assertEquals(null, $this->object->getParam('projectId'));
$this->assertEquals(null, $this->object->getParam('networkRequestSize'));
}
public function testReset(): void
{
$this->object
->setParam('projectId', 'appwrite_test')
->setParam('networkRequestSize', 100)
;
$this->assertEquals('appwrite_test', $this->object->getParam('projectId'));
$this->assertEquals(100, $this->object->getParam('networkRequestSize'));
$this->object->reset();
$this->assertEquals(null, $this->object->getParam('projectId'));
$this->assertEquals(null, $this->object->getParam('networkRequestSize'));
$this->assertEquals('appwrite.usage', $this->object->getNamespace());
$result = $this->client->enqueue($inToQueue);
$this->assertTrue($result);
$outFromQueue = $this->connection->leftPopArray('utopia-queue.queue.' . self::QUEUE_NAME, 0)['payload'];
$this->assertNotEmpty($outFromQueue);
$this->assertSame($inToQueue, $outFromQueue);
}
}

1
tests/unit/Utopia/Response/Filters/V15Test.php
View file

@ -7,7 +7,6 @@ use Appwrite\Utopia\Response;
use Utopia\Database\Helpers\Permission;
use Utopia\Database\Helpers\Role;
use PHPUnit\Framework\TestCase;
use stdClass;
class V15Test extends TestCase
{