CodeMirror/appwrite
1
0
Fork 0
mirror of synced 2024-10-02 02:07:04 +13:00
Code Issues Projects Releases Wiki Activity

Merge branch 'main' of github.com:appwrite/appwrite into remove-cloud-scripts

Browse source 
 Conflicts:
	Dockerfile
	src/Appwrite/Platform/Services/Workers.php
This commit is contained in:
shimon 2024-02-13 10:29:17 +02:00
commit 8c44627da1
35 changed files with 692 additions and 249 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
.env
View file

@ -73,11 +73,12 @@ _APP_EXECUTOR_SECRET=your-secret-key
_APP_EXECUTOR_HOST=http://proxy/v1
_APP_FUNCTIONS_RUNTIMES=php-8.0,node-18.0,python-3.9,ruby-3.1
_APP_MAINTENANCE_INTERVAL=86400
_APP_MAINTENANCE_DELAY=
_APP_MAINTENANCE_RETENTION_CACHE=2592000
_APP_MAINTENANCE_RETENTION_EXECUTION=1209600
_APP_MAINTENANCE_RETENTION_ABUSE=86400
_APP_MAINTENANCE_RETENTION_AUDIT=1209600
_APP_USAGE_AGGREGATION_INTERVAL=60000
_APP_USAGE_AGGREGATION_INTERVAL=20
_APP_MAINTENANCE_RETENTION_USAGE_HOURLY=8640000
_APP_MAINTENANCE_RETENTION_SCHEDULES=86400
_APP_USAGE_STATS=enabled

17
Dockerfile
View file

@ -97,9 +97,24 @@ RUN chmod +x /usr/local/bin/doctor && \
chmod +x /usr/local/bin/worker-mails && \
chmod +x /usr/local/bin/worker-messaging && \
chmod +x /usr/local/bin/worker-webhooks && \
chmod +x /usr/local/bin/worker-migrations
chmod +x /usr/local/bin/worker-migrations && \
chmod +x /usr/local/bin/worker-usage && \
chmod +x /usr/local/bin/worker-usage-dump
# Cloud Executabless
RUN chmod +x /usr/local/bin/hamster && \
chmod +x /usr/local/bin/volume-sync && \
chmod +x /usr/local/bin/patch-delete-schedule-updated-at-attribute && \
chmod +x /usr/local/bin/patch-recreate-repositories-documents && \
chmod +x /usr/local/bin/patch-delete-project-collections && \
chmod +x /usr/local/bin/delete-orphaned-projects && \
chmod +x /usr/local/bin/clear-card-cache && \
chmod +x /usr/local/bin/calc-users-stats && \
chmod +x /usr/local/bin/calc-tier-stats && \
chmod +x /usr/local/bin/get-migration-stats && \
chmod +x /usr/local/bin/create-inf-metric
# Letsencrypt Permissions
RUN mkdir -p /etc/letsencrypt/live/ && chmod -Rf 755 /etc/letsencrypt/live/

21
app/config/errors.php
View file

@ -251,6 +251,11 @@ return [
'description' => 'User phone is already verified',
'code' => 409
],
Exception::USER_DELETION_PROHIBITED => [
'name' => Exception::USER_DELETION_PROHIBITED,
'description' => 'User deletion is not allowed for users with active memberships. Please delete all confirmed memberships before deleting the account.',
'code' => 400
],
/** Teams */
Exception::TEAM_NOT_FOUND => [
@ -768,10 +773,22 @@ return [
],
/** Health */
Exception::QUEUE_SIZE_EXCEEDED => [
'name' => Exception::QUEUE_SIZE_EXCEEDED,
Exception::HEALTH_QUEUE_SIZE_EXCEEDED => [
'name' => Exception::HEALTH_QUEUE_SIZE_EXCEEDED,
'description' => 'Queue size threshold hit.',
'code' => 503,
'publish' => false
],
Exception::HEALTH_CERTIFICATE_EXPIRED => [
'name' => Exception::HEALTH_CERTIFICATE_EXPIRED,
'description' => 'The SSL certificate for the specified domain has expired and is no longer valid.',
'code' => 404,
],
Exception::HEALTH_INVALID_HOST => [
'name' => Exception::HEALTH_INVALID_HOST,
'description' => 'Failed to establish a connection to the specified domain. Please verify the domain name and ensure that the server is running and accessible.',
'code' => 404,
],
];

9
app/config/variables.php
View file

@ -948,6 +948,15 @@ return [
'question' => '',
'filter' => ''
],
[
'name' => '_APP_MAINTENANCE_DELAY',
'description' => 'Delay value containing the number of seconds that the Appwrite maintenance process should wait before executing system cleanups and optimizations. The default value is 0 seconds.',
'introduction' => '1.5.0',
'default' => '0',
'required' => false,
'question' => '',
'filter' => ''
],
[
'name' => '_APP_MAINTENANCE_RETENTION_CACHE',
'description' => 'The maximum duration (in seconds) upto which to retain cached files. The default value is 2592000 seconds (30 days).',

20
app/controllers/api/account.php
View file

@ -1336,8 +1336,6 @@ App::post('/v1/account/sessions/phone')
$message = $message->setParam('{{token}}', $secret);
$message = $message->render();
var_dump($request->getIP());
var_dump($project->getId());
$queueForMessaging
->setRecipient($phone)
->setMessage($message)
@ -1793,7 +1791,7 @@ App::get('/v1/account/logs')
}
$response->dynamic(new Document([
'total' => $audit->countLogsByUser($user->getId()),
'total' => $audit->countLogsByUser($user->getInternalId()),
'logs' => $output,
]), Response::MODEL_LOG_LIST);
});
@ -2939,8 +2937,6 @@ App::post('/v1/account/verification/phone')
$message = $message->setParam('{{token}}', $secret);
$message = $message->render();
var_dump($request->getIP());
var_dump($project->getId());
$queueForMessaging
->setRecipient($user->getAttribute('phone'))
->setMessage($message)
@ -3038,15 +3034,27 @@ App::delete('/v1/account')
->label('sdk.response.code', Response::STATUS_CODE_NOCONTENT)
->label('sdk.response.model', Response::MODEL_NONE)
->inject('user')
->inject('project')
->inject('response')
->inject('dbForProject')
->inject('queueForEvents')
->inject('queueForDeletes')
->action(function (Document $user, Response $response, Database $dbForProject, Event $queueForEvents, Delete $queueForDeletes) {
->action(function (Document $user, Document $project, Response $response, Database $dbForProject, Event $queueForEvents, Delete $queueForDeletes) {
if ($user->isEmpty()) {
throw new Exception(Exception::USER_NOT_FOUND);
}
if ($project->getId() === 'console') {
// get all memberships
$memberships = $user->getAttribute('memberships', []);
foreach ($memberships as $membership) {
// prevent deletion if at least one active membership
if ($membership->getAttribute('confirm', false)) {
throw new Exception(Exception::USER_DELETION_PROHIBITED);
}
}
}
$dbForProject->deleteDocument('users', $user->getId());
$queueForDeletes

77
app/controllers/api/health.php
View file

@ -7,6 +7,7 @@ use Appwrite\Utopia\Response;
use Utopia\App;
use Utopia\Config\Config;
use Utopia\Database\Document;
use Utopia\Domains\Validator\PublicDomain;
use Utopia\Pools\Group;
use Utopia\Queue\Client;
use Utopia\Queue\Connection;
@ -14,7 +15,9 @@ use Utopia\Registry\Registry;
use Utopia\Storage\Device;
use Utopia\Storage\Device\Local;
use Utopia\Storage\Storage;
use Utopia\Validator\Domain;
use Utopia\Validator\Integer;
use Utopia\Validator\Multiple;
use Utopia\Validator\Text;
use Utopia\Validator\WhiteList;
@ -356,7 +359,7 @@ App::get('/v1/health/queue/webhooks')
$size = $client->getQueueSize();
if ($size >= $threshold) {
throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
throw new Exception(Exception::HEALTH_QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
}
$response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE);
@ -383,12 +386,62 @@ App::get('/v1/health/queue/logs')
$size = $client->getQueueSize();
if ($size >= $threshold) {
throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
throw new Exception(Exception::HEALTH_QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
}
$response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE);
}, ['response']);
App::get('/v1/health/certificate')
->desc('Get the SSL certificate for a domain')
->groups(['api', 'health'])
->label('scope', 'health.read')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'health')
->label('sdk.method', 'getCertificate')
->label('sdk.description', '/docs/references/health/get-certificate.md')
->label('sdk.response.code', Response::STATUS_CODE_OK)
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
->label('sdk.response.model', Response::MODEL_HEALTH_CERTIFICATE)
->param('domain', null, new Multiple([new Domain(), new PublicDomain()]), Multiple::TYPE_STRING, 'Domain name')
->inject('response')
->action(function (string $domain, Response $response) {
if (filter_var($domain, FILTER_VALIDATE_URL)) {
$domain = parse_url($domain, PHP_URL_HOST);
}
$sslContext = stream_context_create([
"ssl" => [
"capture_peer_cert" => true
]
]);
$sslSocket = stream_socket_client("ssl://" . $domain . ":443", $errno, $errstr, 30, STREAM_CLIENT_CONNECT, $sslContext);
if (!$sslSocket) {
throw new Exception(Exception::HEALTH_INVALID_HOST);
}
$streamContextParams = stream_context_get_params($sslSocket);
$peerCertificate = $streamContextParams['options']['ssl']['peer_certificate'];
$certificatePayload = openssl_x509_parse($peerCertificate);
$sslExpiration = $certificatePayload['validTo_time_t'];
$status = $sslExpiration < time() ? 'fail' : 'pass';
if ($status === 'fail') {
throw new Exception(Exception::HEALTH_CERTIFICATE_EXPIRED);
}
$response->dynamic(new Document([
'name' => $certificatePayload['name'],
'subjectSN' => $certificatePayload['subject']['CN'],
'issuerOrganisation' => $certificatePayload['issuer']['O'],
'validFrom' => $certificatePayload['validFrom_time_t'],
'validTo' => $certificatePayload['validTo_time_t'],
'signatureTypeSN' => $certificatePayload['signatureTypeSN'],
]), Response::MODEL_HEALTH_CERTIFICATE);
}, ['response']);
App::get('/v1/health/queue/certificates')
->desc('Get certificates queue')
->groups(['api', 'health'])
@ -410,7 +463,7 @@ App::get('/v1/health/queue/certificates')
$size = $client->getQueueSize();
if ($size >= $threshold) {
throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
throw new Exception(Exception::HEALTH_QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
}
$response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE);
@ -437,7 +490,7 @@ App::get('/v1/health/queue/builds')
$size = $client->getQueueSize();
if ($size >= $threshold) {
throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
throw new Exception(Exception::HEALTH_QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
}
$response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE);
@ -465,7 +518,7 @@ App::get('/v1/health/queue/databases')
$size = $client->getQueueSize();
if ($size >= $threshold) {
throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
throw new Exception(Exception::HEALTH_QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
}
$response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE);
@ -492,7 +545,7 @@ App::get('/v1/health/queue/deletes')
$size = $client->getQueueSize();
if ($size >= $threshold) {
throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
throw new Exception(Exception::HEALTH_QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
}
$response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE);
@ -519,7 +572,7 @@ App::get('/v1/health/queue/mails')
$size = $client->getQueueSize();
if ($size >= $threshold) {
throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
throw new Exception(Exception::HEALTH_QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
}
$response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE);
@ -546,7 +599,7 @@ App::get('/v1/health/queue/messaging')
$size = $client->getQueueSize();
if ($size >= $threshold) {
throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
throw new Exception(Exception::HEALTH_QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
}
$response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE);
@ -573,7 +626,7 @@ App::get('/v1/health/queue/migrations')
$size = $client->getQueueSize();
if ($size >= $threshold) {
throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
throw new Exception(Exception::HEALTH_QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
}
$response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE);
@ -600,7 +653,7 @@ App::get('/v1/health/queue/functions')
$size = $client->getQueueSize();
if ($size >= $threshold) {
throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
throw new Exception(Exception::HEALTH_QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
}
$response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE);
@ -695,7 +748,7 @@ App::get('/v1/health/queue/failed/:name')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'health')
->label('sdk.method', 'getFailedJobs')
->param('queueName', '', new WhiteList([
->param('name', '', new WhiteList([
Event::DATABASE_QUEUE_NAME,
Event::DELETE_QUEUE_NAME,
Event::AUDITS_QUEUE_NAME,
@ -723,7 +776,7 @@ App::get('/v1/health/queue/failed/:name')
$failed = $client->countFailedJobs();
if ($failed >= $threshold) {
throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue failed jobs threshold hit. Current size is {$failed} and threshold is {$threshold}.");
throw new Exception(Exception::HEALTH_QUEUE_SIZE_EXCEEDED, "Queue failed jobs threshold hit. Current size is {$failed} and threshold is {$threshold}.");
}
$response->dynamic(new Document([ 'size' => $failed ]), Response::MODEL_HEALTH_QUEUE);

10
app/controllers/api/teams.php
View file

@ -380,7 +380,6 @@ App::post('/v1/teams/:teamId/memberships')
->param('roles', [], new ArrayList(new Key(), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Array of strings. Use this param to set the user roles in the team. A role can be any string. Learn more about [roles and permissions](https://appwrite.io/docs/permissions). Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' roles are allowed, each 32 characters long.')
->param('url', '', fn($clients) => new Host($clients), 'URL to redirect the user back to your app from the invitation email. Only URLs from hostnames in your project platform list are allowed. This requirement helps to prevent an [open redirect](https://cheatsheetseries.owasp.org/cheatsheets/Unvalidated_Redirects_and_Forwards_Cheat_Sheet.html) attack against your project API.', true, ['clients']) // TODO add our own built-in confirm page
->param('name', '', new Text(128), 'Name of the new team member. Max length: 128 chars.', true)
->inject('request')
->inject('response')
->inject('project')
->inject('user')
@ -389,7 +388,7 @@ App::post('/v1/teams/:teamId/memberships')
->inject('queueForMails')
->inject('queueForMessaging')
->inject('queueForEvents')
->action(function (string $teamId, string $email, string $userId, string $phone, array $roles, string $url, string $name, Request $request, Response $response, Document $project, Document $user, Database $dbForProject, Locale $locale, Mail $queueForMails, EventPhone $queueForMessaging, Event $queueForEvents) {
->action(function (string $teamId, string $email, string $userId, string $phone, array $roles, string $url, string $name, Response $response, Document $project, Document $user, Database $dbForProject, Locale $locale, Mail $queueForMails, EventPhone $queueForMessaging, Event $queueForEvents) {
$isAPIKey = Auth::isAppUser(Authorization::getRoles());
$isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles());
@ -639,8 +638,6 @@ App::post('/v1/teams/:teamId/memberships')
$message = $message->setParam('{{token}}', $url);
$message = $message->render();
var_dump($request->getIP());
var_dump($project->getId());
$queueForMessaging
->setRecipient($phone)
->setMessage($message)
@ -970,7 +967,7 @@ App::patch('/v1/teams/:teamId/memberships/:membershipId/status')
$dbForProject->deleteCachedDocument('users', $user->getId());
$team = Authorization::skip(fn() => $dbForProject->updateDocument('teams', $team->getId(), $team->setAttribute('total', $team->getAttribute('total', 0) + 1)));
Authorization::skip(fn() => $dbForProject->increaseDocumentAttribute('teams', $team->getId(), 'total', 1));
$queueForEvents
->setParam('teamId', $team->getId())
@ -1050,8 +1047,7 @@ App::delete('/v1/teams/:teamId/memberships/:membershipId')
$dbForProject->deleteCachedDocument('users', $user->getId());
if ($membership->getAttribute('confirm')) { // Count only confirmed members
$team->setAttribute('total', \max($team->getAttribute('total', 0) - 1, 0));
Authorization::skip(fn() => $dbForProject->updateDocument('teams', $team->getId(), $team));
Authorization::skip(fn() => $dbForProject->decreaseDocumentAttribute('teams', $team->getId(), 'total', 1, 0));
}
$queueForEvents

7
app/controllers/api/users.php
View file

@ -9,6 +9,7 @@ use Appwrite\Event\Event;
use Appwrite\Network\Validator\Email;
use Appwrite\Utopia\Database\Validator\CustomId;
use Appwrite\Utopia\Database\Validator\Queries\Identities;
use Utopia\CLI\Console;
use Utopia\Database\Validator\Queries;
use Appwrite\Utopia\Database\Validator\Queries\Users;
use Utopia\Database\Validator\Query\Limit;
@ -141,7 +142,6 @@ App::post('/v1/users')
->inject('hooks')
->action(function (string $userId, ?string $email, ?string $phone, ?string $password, string $name, Response $response, Document $project, Database $dbForProject, Event $queueForEvents, Hooks $hooks) {
$user = createUser('plaintext', '{}', $userId, $email, $password, $phone, $name, $project, $dbForProject, $queueForEvents, $hooks);
$response
->setStatusCode(Response::STATUS_CODE_CREATED)
->dynamic($user, Response::MODEL_USER);
@ -615,6 +615,9 @@ App::get('/v1/users/:userId/logs')
$output[$i] = new Document([
'event' => $log['event'],
'userId' => ID::custom($log['data']['userId']),
'userEmail' => $log['data']['userEmail'] ?? null,
'userName' => $log['data']['userName'] ?? null,
'ip' => $log['ip'],
'time' => $log['time'],
'osCode' => $os['osCode'],
@ -643,7 +646,7 @@ App::get('/v1/users/:userId/logs')
}
$response->dynamic(new Document([
'total' => $audit->countLogsByUser($user->getId()),
'total' => $audit->countLogsByUser($user->getInternalId()),
'logs' => $output,
]), Response::MODEL_LOG_LIST);
});

3
app/controllers/general.php
View file

@ -607,10 +607,11 @@ App::error()
->action(function (Throwable $error, App $utopia, Request $request, Response $response, Document $project, ?Logger $logger, Log $log) {
$version = App::getEnv('_APP_VERSION', 'UNKNOWN');
$route = $utopia->getRoute();
$publish = true;
if ($error instanceof AppwriteException) {
$publish = $error->isPublishable();
} else {
$publish = $error->getCode() === 0 || $error->getCode() >= 500;
}
if ($logger && ($publish || $error->getCode() === 0)) {

1
app/init.php
View file

@ -195,6 +195,7 @@ const FUNCTION_ALLOWLIST_HEADERS_RESPONSE = ['content-type', 'content-length'];
// Usage metrics
const METRIC_TEAMS = 'teams';
const METRIC_USERS = 'users';
const METRIC_MESSAGES = 'messages';
const METRIC_SESSIONS = 'sessions';
const METRIC_DATABASES = 'databases';
const METRIC_COLLECTIONS = 'collections';

1
app/views/install/compose.phtml
View file

@ -143,6 +143,7 @@ services:
- _APP_LOGGING_PROVIDER
- _APP_LOGGING_CONFIG
- _APP_MAINTENANCE_INTERVAL
- _APP_MAINTENANCE_DELAY
- _APP_MAINTENANCE_RETENTION_EXECUTION
- _APP_MAINTENANCE_RETENTION_CACHE
- _APP_MAINTENANCE_RETENTION_ABUSE

15
app/worker.php
View file

@ -14,6 +14,7 @@ use Appwrite\Event\Mail;
use Appwrite\Event\Migration;
use Appwrite\Event\Phone;
use Appwrite\Event\Usage;
use Appwrite\Event\UsageDump;
use Appwrite\Platform\Appwrite;
use Swoole\Runtime;
use Utopia\App;
@ -146,6 +147,9 @@ Server::setResource('log', fn() => new Log());
Server::setResource('queueForUsage', function (Connection $queue) {
return new Usage($queue);
}, ['queue']);
Server::setResource('queueForUsageDump', function (Connection $queue) {
return new UsageDump($queue);
}, ['queue']);
Server::setResource('queue', function (Group $pools) {
return $pools->get('queue')->pop()->getResource();
}, ['pools']);
@ -296,12 +300,9 @@ $worker
Console::error('[Error] Line: ' . $error->getLine());
});
try {
$workerStart = $worker->getWorkerStart();
} catch (\Throwable $error) {
$worker->workerStart();
} finally {
Console::info("Worker $workerName started");
}
$worker->workerStart()
->action(function () use ($workerName) {
Console::info("Worker $workerName started");
});
$worker->start();

3
bin/worker-usage-dump Normal file
View file

@ -0,0 +1,3 @@
#!/bin/sh
php /usr/src/code/app/worker.php usage-dump $@

2
composer.json
View file

@ -50,7 +50,7 @@
"utopia-php/cli": "0.15.*",
"utopia-php/config": "0.2.*",
"utopia-php/database": "0.45.*",
"utopia-php/domains": "0.3.*",
"utopia-php/domains": "0.5.*",
"utopia-php/dsn": "0.1.*",
"utopia-php/framework": "0.33.*",
"utopia-php/image": "0.5.*",

95
composer.lock generated
View file

@ -4,7 +4,7 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
"This file is @generated automatically"
],
"content-hash": "35dcde03d0eb9a0d27de653b9fa038d4",
"content-hash": "fd03f97115d752d1a94b533ccf570109",
"packages": [
{
"name": "adhocore/jwt",
@ -815,16 +815,16 @@
},
{
"name": "symfony/polyfill-php80",
"version": "v1.28.0",
"version": "v1.29.0",
"source": {
"type": "git",
"url": "https://github.com/symfony/polyfill-php80.git",
"reference": "6caa57379c4aec19c0a12a38b59b26487dcfe4b5"
"reference": "87b68208d5c1188808dd7839ee1e6c8ec3b02f1b"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/symfony/polyfill-php80/zipball/6caa57379c4aec19c0a12a38b59b26487dcfe4b5",
"reference": "6caa57379c4aec19c0a12a38b59b26487dcfe4b5",
"url": "https://api.github.com/repos/symfony/polyfill-php80/zipball/87b68208d5c1188808dd7839ee1e6c8ec3b02f1b",
"reference": "87b68208d5c1188808dd7839ee1e6c8ec3b02f1b",
"shasum": ""
},
"require": {
@ -832,9 +832,6 @@
},
"type": "library",
"extra": {
"branch-alias": {
"dev-main": "1.28-dev"
},
"thanks": {
"name": "symfony/polyfill",
"url": "https://github.com/symfony/polyfill"
@ -878,7 +875,7 @@
"shim"
],
"support": {
"source": "https://github.com/symfony/polyfill-php80/tree/v1.28.0"
"source": "https://github.com/symfony/polyfill-php80/tree/v1.29.0"
},
"funding": [
{
@ -894,7 +891,7 @@
"type": "tidelift"
}
],
"time": "2023-01-26T09:26:14+00:00"
"time": "2024-01-29T20:11:03+00:00"
},
{
"name": "utopia-php/abuse",
@ -1190,16 +1187,16 @@
},
{
"name": "utopia-php/database",
"version": "0.45.5",
"version": "0.45.6",
"source": {
"type": "git",
"url": "https://github.com/utopia-php/database.git",
"reference": "0b66a017f817a910acb83e6aea92bccea9571fe6"
"reference": "c7cc6d57683a4c13d9772dbeea343adb72c443fd"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/utopia-php/database/zipball/0b66a017f817a910acb83e6aea92bccea9571fe6",
"reference": "0b66a017f817a910acb83e6aea92bccea9571fe6",
"url": "https://api.github.com/repos/utopia-php/database/zipball/c7cc6d57683a4c13d9772dbeea343adb72c443fd",
"reference": "c7cc6d57683a4c13d9772dbeea343adb72c443fd",
"shasum": ""
},
"require": {
@ -1240,22 +1237,22 @@
],
"support": {
"issues": "https://github.com/utopia-php/database/issues",
"source": "https://github.com/utopia-php/database/tree/0.45.5"
"source": "https://github.com/utopia-php/database/tree/0.45.6"
},
"time": "2024-01-08T17:08:15+00:00"
"time": "2024-02-01T02:33:43+00:00"
},
{
"name": "utopia-php/domains",
"version": "0.3.2",
"version": "0.5.0",
"source": {
"type": "git",
"url": "https://github.com/utopia-php/domains.git",
"reference": "aaa8c9a96c69ccb397997b1f4f2299c66f77eefb"
"reference": "bf07f60326f8389f378ddf6fcde86217e5cfe18c"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/utopia-php/domains/zipball/aaa8c9a96c69ccb397997b1f4f2299c66f77eefb",
"reference": "aaa8c9a96c69ccb397997b1f4f2299c66f77eefb",
"url": "https://api.github.com/repos/utopia-php/domains/zipball/bf07f60326f8389f378ddf6fcde86217e5cfe18c",
"reference": "bf07f60326f8389f378ddf6fcde86217e5cfe18c",
"shasum": ""
},
"require": {
@ -1300,9 +1297,9 @@
],
"support": {
"issues": "https://github.com/utopia-php/domains/issues",
"source": "https://github.com/utopia-php/domains/tree/0.3.2"
"source": "https://github.com/utopia-php/domains/tree/0.5.0"
},
"time": "2023-07-19T16:39:24+00:00"
"time": "2024-01-03T22:04:27+00:00"
},
{
"name": "utopia-php/dsn",
@ -1353,16 +1350,16 @@
},
{
"name": "utopia-php/framework",
"version": "0.33.1",
"version": "0.33.2",
"source": {
"type": "git",
"url": "https://github.com/utopia-php/http.git",
"reference": "b745607aa1875554a0ad52e28f6db918da1ce11c"
"reference": "b1423ca3e3b61c6c4c2e619d2cb80672809a19f3"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/utopia-php/http/zipball/b745607aa1875554a0ad52e28f6db918da1ce11c",
"reference": "b745607aa1875554a0ad52e28f6db918da1ce11c",
"url": "https://api.github.com/repos/utopia-php/http/zipball/b1423ca3e3b61c6c4c2e619d2cb80672809a19f3",
"reference": "b1423ca3e3b61c6c4c2e619d2cb80672809a19f3",
"shasum": ""
},
"require": {
@ -1392,9 +1389,9 @@
],
"support": {
"issues": "https://github.com/utopia-php/http/issues",
"source": "https://github.com/utopia-php/http/tree/0.33.1"
"source": "https://github.com/utopia-php/http/tree/0.33.2"
},
"time": "2024-01-17T16:48:32+00:00"
"time": "2024-01-31T10:35:59+00:00"
},
{
"name": "utopia-php/image",
@ -2471,16 +2468,16 @@
},
{
"name": "doctrine/deprecations",
"version": "1.1.2",
"version": "1.1.3",
"source": {
"type": "git",
"url": "https://github.com/doctrine/deprecations.git",
"reference": "4f2d4f2836e7ec4e7a8625e75c6aa916004db931"
"reference": "dfbaa3c2d2e9a9df1118213f3b8b0c597bb99fab"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/doctrine/deprecations/zipball/4f2d4f2836e7ec4e7a8625e75c6aa916004db931",
"reference": "4f2d4f2836e7ec4e7a8625e75c6aa916004db931",
"url": "https://api.github.com/repos/doctrine/deprecations/zipball/dfbaa3c2d2e9a9df1118213f3b8b0c597bb99fab",
"reference": "dfbaa3c2d2e9a9df1118213f3b8b0c597bb99fab",
"shasum": ""
},
"require": {
@ -2512,9 +2509,9 @@
"homepage": "https://www.doctrine-project.org/",
"support": {
"issues": "https://github.com/doctrine/deprecations/issues",
"source": "https://github.com/doctrine/deprecations/tree/1.1.2"
"source": "https://github.com/doctrine/deprecations/tree/1.1.3"
},
"time": "2023-09-27T20:04:15+00:00"
"time": "2024-01-30T19:34:25+00:00"
},
{
"name": "doctrine/instantiator",
@ -4772,16 +4769,16 @@
},
{
"name": "symfony/polyfill-ctype",
"version": "v1.28.0",
"version": "v1.29.0",
"source": {
"type": "git",
"url": "https://github.com/symfony/polyfill-ctype.git",
"reference": "ea208ce43cbb04af6867b4fdddb1bdbf84cc28cb"
"reference": "ef4d7e442ca910c4764bce785146269b30cb5fc4"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/symfony/polyfill-ctype/zipball/ea208ce43cbb04af6867b4fdddb1bdbf84cc28cb",
"reference": "ea208ce43cbb04af6867b4fdddb1bdbf84cc28cb",
"url": "https://api.github.com/repos/symfony/polyfill-ctype/zipball/ef4d7e442ca910c4764bce785146269b30cb5fc4",
"reference": "ef4d7e442ca910c4764bce785146269b30cb5fc4",
"shasum": ""
},
"require": {
@ -4795,9 +4792,6 @@
},
"type": "library",
"extra": {
"branch-alias": {
"dev-main": "1.28-dev"
},
"thanks": {
"name": "symfony/polyfill",
"url": "https://github.com/symfony/polyfill"
@ -4834,7 +4828,7 @@
"portable"
],
"support": {
"source": "https://github.com/symfony/polyfill-ctype/tree/v1.28.0"
"source": "https://github.com/symfony/polyfill-ctype/tree/v1.29.0"
},
"funding": [
{
@ -4850,20 +4844,20 @@
"type": "tidelift"
}
],
"time": "2023-01-26T09:26:14+00:00"
"time": "2024-01-29T20:11:03+00:00"
},
{
"name": "symfony/polyfill-mbstring",
"version": "v1.28.0",
"version": "v1.29.0",
"source": {
"type": "git",
"url": "https://github.com/symfony/polyfill-mbstring.git",
"reference": "42292d99c55abe617799667f454222c54c60e229"
"reference": "9773676c8a1bb1f8d4340a62efe641cf76eda7ec"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/symfony/polyfill-mbstring/zipball/42292d99c55abe617799667f454222c54c60e229",
"reference": "42292d99c55abe617799667f454222c54c60e229",
"url": "https://api.github.com/repos/symfony/polyfill-mbstring/zipball/9773676c8a1bb1f8d4340a62efe641cf76eda7ec",
"reference": "9773676c8a1bb1f8d4340a62efe641cf76eda7ec",
"shasum": ""
},
"require": {
@ -4877,9 +4871,6 @@
},
"type": "library",
"extra": {
"branch-alias": {
"dev-main": "1.28-dev"
},
"thanks": {
"name": "symfony/polyfill",
"url": "https://github.com/symfony/polyfill"
@ -4917,7 +4908,7 @@
"shim"
],
"support": {
"source": "https://github.com/symfony/polyfill-mbstring/tree/v1.28.0"
"source": "https://github.com/symfony/polyfill-mbstring/tree/v1.29.0"
},
"funding": [
{
@ -4933,7 +4924,7 @@
"type": "tidelift"
}
],
"time": "2023-07-28T09:04:16+00:00"
"time": "2024-01-29T20:11:03+00:00"
},
{
"name": "textalk/websocket",

33
docker-compose.yml
View file

@ -645,6 +645,7 @@ services:
- _APP_MAINTENANCE_RETENTION_AUDIT
- _APP_MAINTENANCE_RETENTION_USAGE_HOURLY
- _APP_MAINTENANCE_RETENTION_SCHEDULES
- _APP_MAINTENANCE_DELAY
appwrite-worker-usage:
entrypoint: worker-usage
@ -677,6 +678,38 @@ services:
- _APP_LOGGING_CONFIG
- _APP_USAGE_AGGREGATION_INTERVAL
appwrite-worker-usage-dump:
entrypoint: worker-usage-dump
<<: *x-logging
container_name: appwrite-worker-usage-dump
image: appwrite-dev
networks:
- appwrite
volumes:
- ./app:/usr/src/code/app
- ./src:/usr/src/code/src
depends_on:
- redis
- mariadb
environment:
- _APP_ENV
- _APP_WORKER_PER_CORE
- _APP_OPENSSL_KEY_V1
- _APP_DB_HOST
- _APP_DB_PORT
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_USAGE_STATS
- _APP_LOGGING_PROVIDER
- _APP_LOGGING_CONFIG
- _APP_USAGE_AGGREGATION_INTERVAL
appwrite-schedule:
entrypoint: schedule
<<: *x-logging

1
docs/references/health/get-certificate.md Normal file
View file

@ -0,0 +1 @@
Get the SSL certificate for a domain

3
src/Appwrite/Event/Event.php
View file

@ -27,6 +27,9 @@ class Event
public const USAGE_QUEUE_NAME = 'v1-usage';
public const USAGE_CLASS_NAME = 'UsageV1';
public const USAGE_DUMP_QUEUE_NAME = 'v1-usage-dump';
public const USAGE_DUMP_CLASS_NAME = 'UsageDumpV1';
public const WEBHOOK_QUEUE_NAME = 'v1-webhooks';
public const WEBHOOK_CLASS_NAME = 'WebhooksV1';

1
src/Appwrite/Event/Usage.php
View file

@ -2,6 +2,7 @@
namespace Appwrite\Event;
use Utopia\CLI\Console;
use Utopia\Queue\Client;
use Utopia\Queue\Connection;
use Utopia\Database\Document;

47
src/Appwrite/Event/UsageDump.php Normal file
View file

@ -0,0 +1,47 @@
<?php
namespace Appwrite\Event;
use Utopia\Queue\Client;
use Utopia\Queue\Connection;
class UsageDump extends Event
{
protected array $stats;
public function __construct(protected Connection $connection)
{
parent::__construct($connection);
$this
->setQueue(Event::USAGE_DUMP_QUEUE_NAME)
->setClass(Event::USAGE_DUMP_CLASS_NAME);
}
/**
* Add Stats.
*
* @param array $stats
* @return self
*/
public function setStats(array $stats): self
{
$this->stats = $stats;
return $this;
}
/**
* Sends metrics to the usage worker.
*
* @return string|bool
*/
public function trigger(): string|bool
{
$client = new Client($this->queue, $this->connection);
return $client->enqueue([
'stats' => $this->stats,
]);
}
}

9
src/Appwrite/Extend/Exception.php
View file

@ -84,8 +84,9 @@ class Exception extends \Exception
public const USER_OAUTH2_BAD_REQUEST = 'user_oauth2_bad_request';
public const USER_OAUTH2_UNAUTHORIZED = 'user_oauth2_unauthorized';
public const USER_OAUTH2_PROVIDER_ERROR = 'user_oauth2_provider_error';
public const USER_EMAIL_ALREADY_VERIFIED = 'user_email_alread_verified';
public const USER_PHONE_ALREADY_VERIFIED = 'user_phone_already_verified';
public const USER_EMAIL_ALREADY_VERIFIED = 'user_email_alread_verified';
public const USER_PHONE_ALREADY_VERIFIED = 'user_phone_already_verified';
public const USER_DELETION_PROHIBITED = 'user_deletion_prohibited';
/** Teams */
public const TEAM_NOT_FOUND = 'team_not_found';
@ -234,7 +235,9 @@ class Exception extends \Exception
public const REALTIME_POLICY_VIOLATION = 'realtime_policy_violation';
/** Health */
public const QUEUE_SIZE_EXCEEDED = 'queue_size_exceeded';
public const HEALTH_QUEUE_SIZE_EXCEEDED = 'health_queue_size_exceeded';
public const HEALTH_CERTIFICATE_EXPIRED = 'health_certificate_expired';
public const HEALTH_INVALID_HOST = 'health_invalid_host';
protected string $type = '';
protected array $errors = [];

4
src/Appwrite/Platform/Services/Workers.php
View file

@ -13,7 +13,7 @@ use Appwrite\Platform\Workers\Functions;
use Appwrite\Platform\Workers\Builds;
use Appwrite\Platform\Workers\Deletes;
use Appwrite\Platform\Workers\Usage;
use Appwrite\Platform\Workers\UsageHook;
use Appwrite\Platform\Workers\UsageDump;
use Appwrite\Platform\Workers\Migrations;
class Workers extends Service
@ -31,7 +31,7 @@ class Workers extends Service
->addAction(Functions::getName(), new Functions())
->addAction(Builds::getName(), new Builds())
->addAction(Deletes::getName(), new Deletes())
->addAction(UsageHook::getName(), new UsageHook())
->addAction(UsageDump::getName(), new UsageDump())
->addAction(Usage::getName(), new Usage())
->addAction(Migrations::getName(), new Migrations())

3
src/Appwrite/Platform/Tasks/Maintenance.php
View file

@ -36,6 +36,7 @@ class Maintenance extends Action
// # of days in seconds (1 day = 86400s)
$interval = (int) App::getEnv('_APP_MAINTENANCE_INTERVAL', '86400');
$delay = (int) App::getEnv('_APP_MAINTENANCE_DELAY', '0');
$usageStatsRetentionHourly = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_USAGE_HOURLY', '8640000'); //100 days
$cacheRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_CACHE', '2592000'); // 30 days
$schedulesDeletionRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_SCHEDULES', '86400'); // 1 Day
@ -59,7 +60,7 @@ class Maintenance extends Action
$this->renewCertificates($dbForConsole, $queueForCertificates);
$this->notifyDeleteCache($cacheRetention, $queueForDeletes);
$this->notifyDeleteSchedules($schedulesDeletionRetention, $queueForDeletes);
}, $interval);
}, $interval, $delay);
}
protected function foreachProject(Database $dbForConsole, callable $callback): void

17
src/Appwrite/Platform/Tasks/QueueCount.php
View file

@ -58,18 +58,11 @@ class QueueCount extends Action
$queueClient = new Client($name, $queue);
$count = 0;
switch ($type) {
case 'success':
$count = $queueClient->countSuccessfulJobs();
break;
case 'failed':
$count = $queueClient->countFailedJobs();
break;
case 'processing':
$count = $queueClient->countProcessingJobs();
break;
$count = match ($type) {
'success' => $queueClient->countSuccessfulJobs(),
'failed' => $queueClient->countFailedJobs(),
'processing' => $queueClient->countProcessingJobs(),
default => 0
};
Console::log("Queue: '{$name}' has {$count} {$type} jobs.");

4
src/Appwrite/Platform/Workers/Deletes.php
View file

@ -516,8 +516,8 @@ class Deletes extends Action
if ($document->getAttribute('confirm')) { // Count only confirmed members
$teamId = $document->getAttribute('teamId');
$team = $dbForProject->getDocument('teams', $teamId);
if (!$team->isEmpty() && $team->getAttribute('total', 0) > 0) {
$dbForProject->decreaseDocumentAttribute('teams', $teamId, 'total', 1);
if (!$team->isEmpty()) {
$dbForProject->decreaseDocumentAttribute('teams', $teamId, 'total', 1, 0);
}
}
});

29
src/Appwrite/Platform/Workers/Messaging.php
View file

@ -5,6 +5,7 @@ namespace Appwrite\Platform\Workers;
use Exception;
use Utopia\App;
use Utopia\CLI\Console;
use Utopia\Database\Document;
use Utopia\DSN\DSN;
use Utopia\Messaging\Messages\SMS;
use Utopia\Messaging\Adapters\SMS\Mock;
@ -15,6 +16,7 @@ use Utopia\Messaging\Adapters\SMS\Twilio;
use Utopia\Messaging\Adapters\SMS\Vonage;
use Utopia\Platform\Action;
use Utopia\Queue\Message;
use Appwrite\Event\Usage;
class Messaging extends Action
{
@ -43,35 +45,39 @@ class Messaging extends Action
$this
->desc('Messaging worker')
->inject('message')
->callback(fn($message) => $this->action($message));
->inject('queueForUsage')
->callback(fn(Message $message, Usage $queueForUsage) => $this->action($message, $queueForUsage));
}
/**
* @param Message $message
* @param Usage $queueForUsage
* @return void
* @throws Exception
*/
public function action(Message $message): void
public function action(Message $message, Usage $queueForUsage): void
{
$payload = $message->getPayload() ?? [];
if (empty($payload)) {
throw new Exception('Missing payload');
}
if (empty($payload['project'])) {
throw new Exception('Project not set in payload');
}
Console::log($payload['project']['$id']);
$project = new Document($payload['project'] ?? []);
Console::log('Project: ' . $project->getId());
$denyList = App::getEnv('_APP_SMS_PROJECTS_DENY_LIST', '');
$denyList = explode(',', $denyList);
if (in_array($payload['project']['$id'], $denyList)) {
if (in_array($project->getId(), $denyList)) {
Console::error("Project is in the deny list. Skipping ...");
return;
}
if (empty($payload)) {
Console::error('Payload arg not found');
return;
}
if (empty($payload['recipient'])) {
Console::error('Recipient arg not found');
return;
@ -112,6 +118,11 @@ class Messaging extends Action
try {
$sms->send($message);
$queueForUsage
->setProject($project)
->addMetric(METRIC_MESSAGES, 1)
->trigger();
} catch (\Exception $error) {
throw new Exception('Error sending message: ' . $error->getMessage(), 500);
}

57
src/Appwrite/Platform/Workers/Usage.php
View file

@ -3,23 +3,23 @@
namespace Appwrite\Platform\Workers;
use Exception;
use Utopia\App;
use Utopia\CLI\Console;
use Utopia\Database\Database;
use Utopia\Database\DateTime;
use Utopia\Database\Document;
use Utopia\Platform\Action;
use Appwrite\Event\UsageDump;
use Utopia\Queue\Message;
class Usage extends Action
{
protected static array $stats = [];
protected array $periods = [
'1h' => 'Y-m-d H:00',
'1d' => 'Y-m-d 00:00',
'inf' => '0000-00-00 00:00'
];
private array $stats = [];
private int $lastTriggeredTime = 0;
private int $keys = 0;
private const INFINITY_PERIOD = '_inf_';
private const KEYS_THRESHOLD = 10000;
protected const INFINITY_PERIOD = '_inf_';
public static function getName(): string
{
return 'usage';
@ -35,26 +35,31 @@ class Usage extends Action
->desc('Usage worker')
->inject('message')
->inject('getProjectDB')
->callback(function (Message $message, callable $getProjectDB) {
$this->action($message, $getProjectDB);
->inject('queueForUsageDump')
->callback(function (Message $message, callable $getProjectDB, UsageDump $queueForUsageDump) {
$this->action($message, $getProjectDB, $queueForUsageDump);
});
$this->lastTriggeredTime = time();
}
/**
* @param Message $message
* @param callable $getProjectDB
* @param UsageDump $queueForUsageDump
* @return void
* @throws \Utopia\Database\Exception
* @throws Exception
*/
public function action(Message $message, callable $getProjectDB): void
public function action(Message $message, callable $getProjectDB, UsageDump $queueForUsageDump): void
{
$payload = $message->getPayload() ?? [];
if (empty($payload)) {
throw new Exception('Missing payload');
}
//Todo Figure out way to preserve keys when the container is being recreated @shimonewman
$payload = $message->getPayload() ?? [];
$aggregationInterval = (int) App::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '20');
$project = new Document($payload['project'] ?? []);
$projectId = $project->getInternalId();
foreach ($payload['reduce'] ?? [] as $document) {
@ -69,17 +74,35 @@ class Usage extends Action
getProjectDB: $getProjectDB
);
}
self::$stats[$projectId]['project'] = $project;
$this->stats[$projectId]['project'] = $project;
foreach ($payload['metrics'] ?? [] as $metric) {
if (!isset(self::$stats[$projectId]['keys'][$metric['key']])) {
self::$stats[$projectId]['keys'][$metric['key']] = $metric['value'];
$this->keys++;
if (!isset($this->stats[$projectId]['keys'][$metric['key']])) {
$this->stats[$projectId]['keys'][$metric['key']] = $metric['value'];
continue;
}
self::$stats[$projectId]['keys'][$metric['key']] += $metric['value'];
$this->stats[$projectId]['keys'][$metric['key']] += $metric['value'];
}
// if keys crossed threshold or X time passed since the last send and there are some keys in the array ($this->stats)
if (
$this->keys >= self::KEYS_THRESHOLD ||
(time() - $this->lastTriggeredTime > $aggregationInterval && $this->keys > 0)
) {
console::warning('[' . DateTime::now() . '] Aggregated ' . $this->keys . ' keys');
$queueForUsageDump
->setStats($this->stats)
->trigger();
$this->stats = [];
$this->keys = 0;
$this->lastTriggeredTime = time();
}
}
/**
* On Documents that tied by relations like functions>deployments>build || documents>collection>database || buckets>files.
* When we remove a parent document we need to deduct his children aggregation from the project scope.

113
src/Appwrite/Platform/Workers/UsageDump.php Normal file
View file

@ -0,0 +1,113 @@
<?php
namespace Appwrite\Platform\Workers;
use Appwrite\Extend\Exception;
use Utopia\App;
use Utopia\Database\Document;
use Utopia\Database\Exception\Duplicate;
use Utopia\Platform\Action;
use Utopia\CLI\Console;
use Utopia\Database\DateTime;
use Utopia\Queue\Message;
class UsageDump extends Action
{
protected array $stats = [];
protected array $periods = [
'1h' => 'Y-m-d H:00',
'1d' => 'Y-m-d 00:00',
'inf' => '0000-00-00 00:00'
];
public static function getName(): string
{
return 'usage-dump';
}
/**
* @throws \Exception
*/
public function __construct()
{
$this
->inject('message')
->inject('getProjectDB')
->callback(function (Message $message, callable $getProjectDB) {
$this->action($message, $getProjectDB);
})
;
}
/**
* @param Message $message
* @param callable $getProjectDB
* @return void
* @throws Exception
* @throws \Utopia\Database\Exception
*/
public function action(Message $message, callable $getProjectDB): void
{
$payload = $message->getPayload() ?? [];
if (empty($payload)) {
throw new Exception('Missing payload');
}
//Todo rename both usage workers @shimonewman
foreach ($payload['stats'] ?? [] as $stats) {
$project = new Document($stats['project'] ?? []);
$numberOfKeys = !empty($stats['keys']) ? count($stats['keys']) : 0;
if ($numberOfKeys === 0) {
continue;
}
console::log('[' . DateTime::now() . '] ProjectId [' . $project->getInternalId() . '] Database [' . $project['database'] . '] ' . $numberOfKeys . ' keys');
try {
$dbForProject = $getProjectDB($project);
foreach ($stats['keys'] ?? [] as $key => $value) {
if ($value == 0) {
continue;
}
foreach ($this->periods as $period => $format) {
$time = 'inf' === $period ? null : date($format, time());
$id = \md5("{$time}_{$period}_{$key}");
try {
$dbForProject->createDocument('stats_v2', new Document([
'$id' => $id,
'period' => $period,
'time' => $time,
'metric' => $key,
'value' => $value,
'region' => App::getEnv('_APP_REGION', 'default'),
]));
} catch (Duplicate $th) {
if ($value < 0) {
$dbForProject->decreaseDocumentAttribute(
'stats_v2',
$id,
'value',
abs($value)
);
} else {
$dbForProject->increaseDocumentAttribute(
'stats_v2',
$id,
'value',
$value
);
}
}
}
}
} catch (\Exception $e) {
console::error('[' . DateTime::now() . '] project [' . $project->getInternalId() . '] database [' . $project['database'] . '] ' . ' ' . $e->getMessage());
}
}
}
}

103
src/Appwrite/Platform/Workers/UsageHook.php
View file

@ -1,103 +0,0 @@
<?php
namespace Appwrite\Platform\Workers;
use Utopia\App;
use Utopia\Database\Document;
use Utopia\Database\Exception\Duplicate;
use Utopia\Platform\Action;
use Utopia\CLI\Console;
use Swoole\Timer;
use Utopia\Database\DateTime;
class UsageHook extends Usage
{
public static function getName(): string
{
return 'usageHook';
}
public function __construct()
{
$this
->setType(Action::TYPE_WORKER_START)
->inject('register')
->inject('getProjectDB')
->callback(function ($register, callable $getProjectDB) {
$this->action($register, $getProjectDB);
})
;
}
/**
* @param $register
* @param $getProjectDB
* @return void
*/
public function action($register, $getProjectDB): void
{
$interval = (int) App::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '60000');
Timer::tick($interval, function () use ($register, $getProjectDB) {
$offset = count(self::$stats);
$projects = array_slice(self::$stats, 0, $offset, true);
array_splice(self::$stats, 0, $offset);
foreach ($projects as $data) {
$numberOfKeys = !empty($data['keys']) ? count($data['keys']) : 0;
$projectInternalId = $data['project']->getInternalId();
$database = $data['project']['database'] ?? '';
console::warning('Ticker started ' . DateTime::now());
if ($numberOfKeys === 0) {
continue;
}
try {
$dbForProject = $getProjectDB($data['project']);
foreach ($data['keys'] ?? [] as $key => $value) {
if ($value == 0) {
continue;
}
foreach ($this->periods as $period => $format) {
$time = 'inf' === $period ? null : date($format, time());
$id = \md5("{$time}_{$period}_{$key}");
try {
$dbForProject->createDocument('stats_v2', new Document([
'$id' => $id,
'period' => $period,
'time' => $time,
'metric' => $key,
'value' => $value,
'region' => App::getEnv('_APP_REGION', 'default'),
]));
} catch (Duplicate $th) {
if ($value < 0) {
$dbForProject->decreaseDocumentAttribute(
'stats_v2',
$id,
'value',
abs($value)
);
} else {
$dbForProject->increaseDocumentAttribute(
'stats_v2',
$id,
'value',
$value
);
}
}
}
}
} catch (\Exception $e) {
console::error(DateTime::now() . ' ' . $projectInternalId . ' ' . $e->getMessage());
}
}
});
}
}

6
src/Appwrite/Specification/Format/OpenAPI3.php
View file

@ -295,7 +295,7 @@ class OpenAPI3 extends Format
switch ((!empty($validator)) ? \get_class($validator) : '') {
case 'Utopia\Validator\Text':
$node['schema']['type'] = $validator->getType();
$node['schema']['x-example'] = '[' . \strtoupper(Template::fromCamelCaseToSnake($node['name'])) . ']';
$node['schema']['x-example'] = '<' . \strtoupper(Template::fromCamelCaseToSnake($node['name'])) . '>';
break;
case 'Utopia\Validator\Boolean':
$node['schema']['type'] = $validator->getType();
@ -303,14 +303,14 @@ class OpenAPI3 extends Format
break;
case 'Utopia\Database\Validator\UID':
$node['schema']['type'] = $validator->getType();
$node['schema']['x-example'] = '[' . \strtoupper(Template::fromCamelCaseToSnake($node['name'])) . ']';
$node['schema']['x-example'] = '<' . \strtoupper(Template::fromCamelCaseToSnake($node['name'])) . '>';
break;
case 'Appwrite\Utopia\Database\Validator\CustomId':
if ($route->getLabel('sdk.methodType', '') === 'upload') {
$node['schema']['x-upload-id'] = true;
}
$node['schema']['type'] = $validator->getType();
$node['schema']['x-example'] = '[' . \strtoupper(Template::fromCamelCaseToSnake($node['name'])) . ']';
$node['schema']['x-example'] = '<' . \strtoupper(Template::fromCamelCaseToSnake($node['name'])) . '>';
break;
case 'Utopia\Database\Validator\DatetimeValidator':
$node['schema']['type'] = $validator->getType();

6
src/Appwrite/Specification/Format/Swagger2.php
View file

@ -297,7 +297,7 @@ class Swagger2 extends Format
switch ((!empty($validator)) ? \get_class($validator) : '') {
case 'Utopia\Validator\Text':
$node['type'] = $validator->getType();
$node['x-example'] = '[' . \strtoupper(Template::fromCamelCaseToSnake($node['name'])) . ']';
$node['x-example'] = '<' . \strtoupper(Template::fromCamelCaseToSnake($node['name'])) . '>';
break;
case 'Utopia\Validator\Boolean':
$node['type'] = $validator->getType();
@ -308,11 +308,11 @@ class Swagger2 extends Format
$node['x-upload-id'] = true;
}
$node['type'] = $validator->getType();
$node['x-example'] = '[' . \strtoupper(Template::fromCamelCaseToSnake($node['name'])) . ']';
$node['x-example'] = '<' . \strtoupper(Template::fromCamelCaseToSnake($node['name'])) . '>';
break;
case 'Utopia\Database\Validator\UID':
$node['type'] = $validator->getType();
$node['x-example'] = '[' . \strtoupper(Template::fromCamelCaseToSnake($node['name'])) . ']';
$node['x-example'] = '<' . \strtoupper(Template::fromCamelCaseToSnake($node['name'])) . '>';
break;
case 'Utopia\Database\Validator\DatetimeValidator':
$node['type'] = $validator->getType();

3
src/Appwrite/Utopia/Response.php
View file

@ -70,6 +70,7 @@ use Appwrite\Utopia\Response\Model\Token;
use Appwrite\Utopia\Response\Model\Webhook;
use Appwrite\Utopia\Response\Model\Preferences;
use Appwrite\Utopia\Response\Model\HealthAntivirus;
use Appwrite\Utopia\Response\Model\HealthCertificate;
use Appwrite\Utopia\Response\Model\HealthQueue;
use Appwrite\Utopia\Response\Model\HealthStatus;
use Appwrite\Utopia\Response\Model\HealthTime;
@ -253,6 +254,7 @@ class Response extends SwooleResponse
public const MODEL_HEALTH_QUEUE = 'healthQueue';
public const MODEL_HEALTH_TIME = 'healthTime';
public const MODEL_HEALTH_ANTIVIRUS = 'healthAntivirus';
public const MODEL_HEALTH_CERTIFICATE = 'healthCertificate';
public const MODEL_HEALTH_STATUS_LIST = 'healthStatusList';
// Console
@ -390,6 +392,7 @@ class Response extends SwooleResponse
->setModel(new HealthAntivirus())
->setModel(new HealthQueue())
->setModel(new HealthStatus())
->setModel(new HealthCertificate())
->setModel(new HealthTime())
->setModel(new HealthVersion())
->setModel(new Metric())

71
src/Appwrite/Utopia/Response/Model/HealthCertificate.php Normal file
View file

@ -0,0 +1,71 @@
<?php
namespace Appwrite\Utopia\Response\Model;
use Appwrite\Utopia\Response;
use Appwrite\Utopia\Response\Model;
class HealthCertificate extends Model
{
public function __construct()
{
$this
->addRule('name', [
'type' => self::TYPE_STRING,
'description' => 'Certificate name',
'default' => '',
'example' => '/CN=www.google.com',
])
->addRule('subjectSN', [
'type' => self::TYPE_STRING,
'description' => 'Subject SN',
'default' => 'www.google.com',
'example' => '',
])
->addRule('issuerOrganisation', [
'type' => self::TYPE_STRING,
'description' => 'Issuer organisation',
'default' => 'Google Trust Services LLC',
'example' => '',
])
->addRule('validFrom', [
'type' => self::TYPE_STRING,
'description' => 'Valid from',
'default' => '',
'example' => '1704200998',
])
->addRule('validTo', [
'type' => self::TYPE_STRING,
'description' => 'Valid to',
'default' => '',
'example' => '1711458597',
])
->addRule('signatureTypeSN', [
'type' => self::TYPE_STRING,
'description' => 'Signature type SN',
'default' => '',
'example' => 'RSA-SHA256',
])
;
}
/**
* Get Name
*
* @return string
*/
public function getName(): string
{
return 'Health Certificate';
}
/**
* Get Type
*
* @return string
*/
public function getType(): string
{
return Response::MODEL_HEALTH_CERTIFICATE;
}
}

77
tests/e2e/Services/Account/AccountConsoleClientTest.php
View file

@ -2,17 +2,90 @@
namespace Tests\E2E\Services\Account;
use Appwrite\Extend\Exception;
use Tests\E2E\Scopes\Scope;
use Tests\E2E\Scopes\ProjectConsole;
use Tests\E2E\Scopes\SideClient;
use Utopia\Database\Helpers\ID;
use Tests\E2E\Client;
use Utopia\Database\Validator\Datetime as DatetimeValidator;
class AccountConsoleClientTest extends Scope
{
use AccountBase;
use ProjectConsole;
use SideClient;
public function testDeleteAccount(): void
{
$email = uniqid() . 'user@localhost.test';
$password = 'password';
$name = 'User Name';
$response = $this->client->call(Client::METHOD_POST, '/account', array_merge([
'origin' => 'http://localhost',
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
]), [
'userId' => ID::unique(),
'email' => $email,
'password' => $password,
'name' => $name,
]);
$this->assertEquals($response['headers']['status-code'], 201);
$response = $this->client->call(Client::METHOD_POST, '/account/sessions/email', array_merge([
'origin' => 'http://localhost',
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
]), [
'email' => $email,
'password' => $password,
]);
$this->assertEquals($response['headers']['status-code'], 201);
$session = $response['cookies']['a_session_' . $this->getProject()['$id']];
// create team
$team = $this->client->call(Client::METHOD_POST, '/teams', [
'origin' => 'http://localhost',
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'cookie' => 'a_session_' . $this->getProject()['$id'] . '=' . $session,
], [
'teamId' => 'unique()',
'name' => 'myteam'
]);
$this->assertEquals($team['headers']['status-code'], 201);
$teamId = $team['body']['$id'];
$response = $this->client->call(Client::METHOD_DELETE, '/account', array_merge([
'origin' => 'http://localhost',
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'cookie' => 'a_session_' . $this->getProject()['$id'] . '=' . $session,
]));
$this->assertEquals($response['headers']['status-code'], 400);
// DELETE TEAM
$response = $this->client->call(Client::METHOD_DELETE, '/teams/' . $teamId, array_merge([
'origin' => 'http://localhost',
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'cookie' => 'a_session_' . $this->getProject()['$id'] . '=' . $session,
]));
$this->assertEquals($response['headers']['status-code'], 204);
sleep(2);
$response = $this->client->call(Client::METHOD_DELETE, '/account', array_merge([
'origin' => 'http://localhost',
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'cookie' => 'a_session_' . $this->getProject()['$id'] . '=' . $session,
]));
$this->assertEquals($response['headers']['status-code'], 204);
}
}

70
tests/e2e/Services/Health/HealthCustomServerTest.php
View file

@ -424,4 +424,74 @@ class HealthCustomServerTest extends Scope
return [];
}
public function testCertificateValidity(): array
{
/**
* Test for SUCCESS
*/
$response = $this->client->call(Client::METHOD_GET, '/health/certificate?domain=www.google.com', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), []);
$this->assertEquals(200, $response['headers']['status-code']);
$this->assertEquals('/CN=www.google.com', $response['body']['name']);
$this->assertEquals('www.google.com', $response['body']['subjectSN']);
$this->assertEquals('Google Trust Services LLC', $response['body']['issuerOrganisation']);
$this->assertIsInt($response['body']['validFrom']);
$this->assertIsInt($response['body']['validTo']);
$response = $this->client->call(Client::METHOD_GET, '/health/certificate?domain=appwrite.io', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), []);
$this->assertEquals(200, $response['headers']['status-code']);
$this->assertEquals('/CN=appwrite.io', $response['body']['name']);
$this->assertEquals('appwrite.io', $response['body']['subjectSN']);
$this->assertEquals("Let's Encrypt", $response['body']['issuerOrganisation']);
$this->assertIsInt($response['body']['validFrom']);
$this->assertIsInt($response['body']['validTo']);
$response = $this->client->call(Client::METHOD_GET, '/health/certificate?domain=https://google.com', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), []);
$this->assertEquals(200, $response['headers']['status-code']);
/**
* Test for FAILURE
*/
$response = $this->client->call(Client::METHOD_GET, '/health/certificate?domain=localhost', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), []);
$this->assertEquals(400, $response['headers']['status-code']);
$response = $this->client->call(Client::METHOD_GET, '/health/certificate?domain=doesnotexist.com', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), []);
$this->assertEquals(404, $response['headers']['status-code']);
$response = $this->client->call(Client::METHOD_GET, '/health/certificate?domain=www.google.com/usr/src/local', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), []);
$this->assertEquals(400, $response['headers']['status-code']);
$response = $this->client->call(Client::METHOD_GET, '/health/certificate?domain=', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), []);
$this->assertEquals(400, $response['headers']['status-code']);
return [];
}
}