From 18f7168160b69e122aa00a12af402c3a76afbc9f Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Mon, 12 Feb 2024 14:18:19 +1300 Subject: [PATCH] Merge main --- .env | 2 + Dockerfile | 4 + app/config/errors.php | 17 ++- app/config/variables.php | 9 ++ app/controllers/api/account.php | 23 ++-- app/controllers/api/health.php | 115 ++++++++++++++++-- app/controllers/api/proxy.php | 11 +- app/controllers/api/teams.php | 4 +- app/controllers/general.php | 96 +++++++-------- app/controllers/shared/api.php | 11 +- app/controllers/shared/api/auth.php | 8 +- app/http.php | 7 +- app/init.php | 7 +- app/views/install/compose.phtml | 1 + bin/queue-count-failed | 3 + bin/queue-count-processing | 3 + bin/queue-count-success | 3 + bin/queue-retry | 3 + composer.json | 4 +- composer.lock | 57 ++++----- docker-compose.yml | 2 + docs/references/health/get-certificate.md | 1 + .../health/get-failed-queue-jobs.md | 1 + src/Appwrite/Extend/Exception.php | 17 ++- src/Appwrite/Network/Validator/CNAME.php | 14 +++ src/Appwrite/Platform/Services/Tasks.php | 21 ++-- src/Appwrite/Platform/Tasks/Maintenance.php | 3 +- src/Appwrite/Platform/Tasks/QueueCount.php | 70 +++++++++++ src/Appwrite/Platform/Tasks/QueueRetry.php | 64 ++++++++++ src/Appwrite/Platform/Tasks/SSL.php | 10 +- .../Platform/Workers/Certificates.php | 24 +++- src/Appwrite/Platform/Workers/Deletes.php | 7 +- src/Appwrite/Platform/Workers/Messaging.php | 59 ++++++--- src/Appwrite/Platform/Workers/Usage.php | 13 -- src/Appwrite/Platform/Workers/UsageHook.php | 35 ------ src/Appwrite/Utopia/Response.php | 3 + .../Response/Model/HealthCertificate.php | 71 +++++++++++ .../Health/HealthCustomServerTest.php | 70 +++++++++++ 38 files changed, 645 insertions(+), 228 deletions(-) create mode 100644 bin/queue-count-failed create mode 100644 bin/queue-count-processing create mode 100644 bin/queue-count-success create mode 100644 bin/queue-retry create mode 100644 docs/references/health/get-certificate.md create mode 100644 docs/references/health/get-failed-queue-jobs.md create mode 100644 src/Appwrite/Platform/Tasks/QueueCount.php create mode 100644 src/Appwrite/Platform/Tasks/QueueRetry.php create mode 100644 src/Appwrite/Utopia/Response/Model/HealthCertificate.php diff --git a/.env b/.env index c8c0fccb2..0b5ddf24c 100644 --- a/.env +++ b/.env @@ -59,6 +59,7 @@ _APP_SMTP_USERNAME= _APP_SMTP_PASSWORD= _APP_SMS_PROVIDER=sms://username:password@mock _APP_SMS_FROM=+123456789 +_APP_SMS_PROJECTS_DENY_LIST= _APP_STORAGE_LIMIT=30000000 _APP_STORAGE_PREVIEW_LIMIT=20000000 _APP_FUNCTIONS_SIZE_LIMIT=30000000 @@ -73,6 +74,7 @@ _APP_EXECUTOR_SECRET=your-secret-key _APP_EXECUTOR_HOST=http://proxy/v1 _APP_FUNCTIONS_RUNTIMES=php-8.0,node-18.0,python-3.9,ruby-3.1 _APP_MAINTENANCE_INTERVAL=86400 +_APP_MAINTENANCE_DELAY= _APP_MAINTENANCE_RETENTION_CACHE=2592000 _APP_MAINTENANCE_RETENTION_EXECUTION=1209600 _APP_MAINTENANCE_RETENTION_ABUSE=86400 diff --git a/Dockerfile b/Dockerfile index 968b0bb22..de858afe7 100755 --- a/Dockerfile +++ b/Dockerfile @@ -89,6 +89,10 @@ RUN chmod +x /usr/local/bin/doctor && \ chmod +x /usr/local/bin/test && \ chmod +x /usr/local/bin/upgrade && \ chmod +x /usr/local/bin/vars && \ + chmod +x /usr/local/bin/queue-retry && \ + chmod +x /usr/local/bin/queue-count-failed && \ + chmod +x /usr/local/bin/queue-count-processing && \ + chmod +x /usr/local/bin/queue-count-success && \ chmod +x /usr/local/bin/worker-audits && \ chmod +x /usr/local/bin/worker-builds && \ chmod +x /usr/local/bin/worker-certificates && \ diff --git a/app/config/errors.php b/app/config/errors.php index 79a092ec9..8c3b08b96 100644 --- a/app/config/errors.php +++ b/app/config/errors.php @@ -711,6 +711,7 @@ return [ 'name' => Exception::RULE_VERIFICATION_FAILED, 'description' => 'Domain verification failed. Please check if your DNS records are correct and try again.', 'code' => 401, + 'publish' => true ], Exception::PROJECT_SMTP_CONFIG_INVALID => [ 'name' => Exception::PROJECT_SMTP_CONFIG_INVALID, @@ -798,13 +799,25 @@ return [ ], /** Health */ - Exception::QUEUE_SIZE_EXCEEDED => [ - 'name' => Exception::QUEUE_SIZE_EXCEEDED, + Exception::HEALTH_QUEUE_SIZE_EXCEEDED => [ + 'name' => Exception::HEALTH_QUEUE_SIZE_EXCEEDED, 'description' => 'Queue size threshold hit.', 'code' => 503, 'publish' => false ], + Exception::HEALTH_CERTIFICATE_EXPIRED => [ + 'name' => Exception::HEALTH_CERTIFICATE_EXPIRED, + 'description' => 'The SSL certificate for the specified domain has expired and is no longer valid.', + 'code' => 404, + ], + + Exception::HEALTH_INVALID_HOST => [ + 'name' => Exception::HEALTH_INVALID_HOST, + 'description' => 'Failed to establish a connection to the specified domain. Please verify the domain name and ensure that the server is running and accessible.', + 'code' => 404, + ], + /** Providers */ Exception::PROVIDER_NOT_FOUND => [ 'name' => Exception::PROVIDER_NOT_FOUND, diff --git a/app/config/variables.php b/app/config/variables.php index d689c8bb0..5909fbfc6 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -948,6 +948,15 @@ return [ 'question' => '', 'filter' => '' ], + [ + 'name' => '_APP_MAINTENANCE_DELAY', + 'description' => 'Delay value containing the number of seconds that the Appwrite maintenance process should wait before executing system cleanups and optimizations. The default value is 0 seconds.', + 'introduction' => '1.5.0', + 'default' => '0', + 'required' => false, + 'question' => '', + 'filter' => '' + ], [ 'name' => '_APP_MAINTENANCE_RETENTION_CACHE', 'description' => 'The maximum duration (in seconds) upto which to retain cached files. The default value is 2592000 seconds (30 days).', diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index 49eabcbf5..a6db09880 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -978,7 +978,7 @@ App::delete('/v1/account/identities/:identityId') App::post('/v1/account/tokens/magic-url') ->alias('/v1/account/sessions/magic-url') ->desc('Create magic URL token') - ->groups(['api', 'account']) + ->groups(['api', 'account', 'auth']) ->label('scope', 'sessions.write') ->label('auth.type', 'magic-url') ->label('audits.event', 'session.create') @@ -986,14 +986,14 @@ App::post('/v1/account/tokens/magic-url') ->label('audits.userId', '{response.userId}') ->label('sdk.auth', []) ->label('sdk.namespace', 'account') - ->label('sdk.method', ['createMagicURLToken', 'createMagicURLSession']) + ->label('sdk.method', 'createMagicURLToken') ->label('sdk.description', '/docs/references/account/create-token-magic-url.md') ->label('sdk.response.code', Response::STATUS_CODE_CREATED) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_TOKEN) - ->label('abuse-limit', 10) - ->label('abuse-key', 'url:{url},email:{param-email}') - ->param('userId', '', new CustomId(), 'User ID. Choose a custom ID or generate a random ID with `ID.unique()`. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.') + ->label('abuse-limit', 60) + ->label('abuse-key', ['url:{url},email:{param-email}', 'url:{url},ip:{ip}']) + ->param('userId', '', new CustomId(), 'Unique Id. Choose a custom ID or generate a random ID with `ID.unique()`. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.') ->param('email', '', new Email(), 'User email.') ->param('url', '', fn($clients) => new Host($clients), 'URL to redirect the user back to your app from the magic URL login. Only URLs from hostnames in your project platform list are allowed. This requirement helps to prevent an [open redirect](https://cheatsheetseries.owasp.org/cheatsheets/Unvalidated_Redirects_and_Forwards_Cheat_Sheet.html) attack against your project API.', true, ['clients']) ->param('phrase', false, new Boolean(), 'Toggle for security phrase. If enabled, email will be send with a randomly generated phrase and the phrase will also be included in the response. Confirming phrases match increases the security of your authentication flow.', true) @@ -1586,7 +1586,7 @@ App::put('/v1/account/sessions/magic-url') App::post('/v1/account/sessions/token') ->desc('Create session') ->label('event', 'users.[userId].sessions.[sessionId].create') - ->groups(['api', 'account']) + ->groups(['api', 'account', 'auth']) ->label('scope', 'sessions.write') ->label('audits.event', 'session.create') ->label('audits.resource', 'user/{response.userId}') @@ -1623,13 +1623,13 @@ App::post('/v1/account/tokens/phone') ->label('audits.userId', '{response.userId}') ->label('sdk.auth', []) ->label('sdk.namespace', 'account') - ->label('sdk.method', ['createPhoneToken', 'createPhoneSession']) + ->label('sdk.method', 'createPhoneToken') ->label('sdk.description', '/docs/references/account/create-token-phone.md') ->label('sdk.response.code', Response::STATUS_CODE_CREATED) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_TOKEN) ->label('abuse-limit', 10) - ->label('abuse-key', 'url:{url},phone:{param-phone}') + ->label('abuse-key', ['url:{url},phone:{param-phone}', 'url:{url},ip:{ip}']) ->param('userId', '', new CustomId(), 'Unique Id. Choose a custom ID or generate a random ID with `ID.unique()`. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.') ->param('phone', '', new Phone(), 'Phone number. Format this number with a leading \'+\' and a country code, e.g., +16175551212.') ->inject('request') @@ -2736,7 +2736,7 @@ App::post('/v1/account/recovery') ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_TOKEN) ->label('abuse-limit', 10) - ->label('abuse-key', ['url:{url},email:{param-email}', 'ip:{ip}']) + ->label('abuse-key', ['url:{url},email:{param-email}', 'url:{url},ip:{ip}']) ->param('email', '', new Email(), 'User email.') ->param('url', '', fn ($clients) => new Host($clients), 'URL to redirect the user back to your app from the recovery email. Only URLs from hostnames in your project platform list are allowed. This requirement helps to prevent an [open redirect](https://cheatsheetseries.owasp.org/cheatsheets/Unvalidated_Redirects_and_Forwards_Cheat_Sheet.html) attack against your project API.', false, ['clients']) ->inject('request') @@ -3204,8 +3204,9 @@ App::put('/v1/account/verification') App::post('/v1/account/verification/phone') ->desc('Create phone verification') - ->groups(['api', 'account']) + ->groups(['api', 'account', 'auth']) ->label('scope', 'accounts.write') + ->label('auth.type', 'phone') ->label('event', 'users.[userId].verification.[tokenId].create') ->label('audits.event', 'verification.create') ->label('audits.resource', 'user/{response.userId}') @@ -3217,7 +3218,7 @@ App::post('/v1/account/verification/phone') ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_TOKEN) ->label('abuse-limit', 10) - ->label('abuse-key', 'userId:{userId}') + ->label('abuse-key', ['url:{url},userId:{userId}', 'url:{url},ip:{ip}']) ->inject('request') ->inject('response') ->inject('user') diff --git a/app/controllers/api/health.php b/app/controllers/api/health.php index 4f459dd86..a85f9da32 100644 --- a/app/controllers/api/health.php +++ b/app/controllers/api/health.php @@ -7,6 +7,7 @@ use Appwrite\Utopia\Response; use Utopia\App; use Utopia\Config\Config; use Utopia\Database\Document; +use Utopia\Domains\Validator\PublicDomain; use Utopia\Pools\Group; use Utopia\Queue\Client; use Utopia\Queue\Connection; @@ -14,8 +15,11 @@ use Utopia\Registry\Registry; use Utopia\Storage\Device; use Utopia\Storage\Device\Local; use Utopia\Storage\Storage; +use Utopia\Validator\Domain; use Utopia\Validator\Integer; +use Utopia\Validator\Multiple; use Utopia\Validator\Text; +use Utopia\Validator\WhiteList; App::get('/v1/health') ->desc('Get HTTP') @@ -355,7 +359,7 @@ App::get('/v1/health/queue/webhooks') $size = $client->getQueueSize(); if ($size >= $threshold) { - throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); + throw new Exception(Exception::HEALTH_QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); } $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); @@ -382,12 +386,62 @@ App::get('/v1/health/queue/logs') $size = $client->getQueueSize(); if ($size >= $threshold) { - throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); + throw new Exception(Exception::HEALTH_QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); } $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); }, ['response']); +App::get('/v1/health/certificate') + ->desc('Get the SSL certificate for a domain') + ->groups(['api', 'health']) + ->label('scope', 'health.read') + ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) + ->label('sdk.namespace', 'health') + ->label('sdk.method', 'getCertificate') + ->label('sdk.description', '/docs/references/health/get-certificate.md') + ->label('sdk.response.code', Response::STATUS_CODE_OK) + ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) + ->label('sdk.response.model', Response::MODEL_HEALTH_CERTIFICATE) + ->param('domain', null, new Multiple([new Domain(), new PublicDomain()]), Multiple::TYPE_STRING, 'Domain name') + ->inject('response') + ->action(function (string $domain, Response $response) { + if (filter_var($domain, FILTER_VALIDATE_URL)) { + $domain = parse_url($domain, PHP_URL_HOST); + } + + $sslContext = stream_context_create([ + "ssl" => [ + "capture_peer_cert" => true + ] + ]); + $sslSocket = stream_socket_client("ssl://" . $domain . ":443", $errno, $errstr, 30, STREAM_CLIENT_CONNECT, $sslContext); + if (!$sslSocket) { + throw new Exception(Exception::HEALTH_INVALID_HOST); + } + + $streamContextParams = stream_context_get_params($sslSocket); + $peerCertificate = $streamContextParams['options']['ssl']['peer_certificate']; + $certificatePayload = openssl_x509_parse($peerCertificate); + + + $sslExpiration = $certificatePayload['validTo_time_t']; + $status = $sslExpiration < time() ? 'fail' : 'pass'; + + if ($status === 'fail') { + throw new Exception(Exception::HEALTH_CERTIFICATE_EXPIRED); + } + + $response->dynamic(new Document([ + 'name' => $certificatePayload['name'], + 'subjectSN' => $certificatePayload['subject']['CN'], + 'issuerOrganisation' => $certificatePayload['issuer']['O'], + 'validFrom' => $certificatePayload['validFrom_time_t'], + 'validTo' => $certificatePayload['validTo_time_t'], + 'signatureTypeSN' => $certificatePayload['signatureTypeSN'], + ]), Response::MODEL_HEALTH_CERTIFICATE); + }, ['response']); + App::get('/v1/health/queue/certificates') ->desc('Get certificates queue') ->groups(['api', 'health']) @@ -409,7 +463,7 @@ App::get('/v1/health/queue/certificates') $size = $client->getQueueSize(); if ($size >= $threshold) { - throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); + throw new Exception(Exception::HEALTH_QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); } $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); @@ -436,7 +490,7 @@ App::get('/v1/health/queue/builds') $size = $client->getQueueSize(); if ($size >= $threshold) { - throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); + throw new Exception(Exception::HEALTH_QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); } $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); @@ -464,7 +518,7 @@ App::get('/v1/health/queue/databases') $size = $client->getQueueSize(); if ($size >= $threshold) { - throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); + throw new Exception(Exception::HEALTH_QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); } $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); @@ -491,7 +545,7 @@ App::get('/v1/health/queue/deletes') $size = $client->getQueueSize(); if ($size >= $threshold) { - throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); + throw new Exception(Exception::HEALTH_QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); } $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); @@ -518,7 +572,7 @@ App::get('/v1/health/queue/mails') $size = $client->getQueueSize(); if ($size >= $threshold) { - throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); + throw new Exception(Exception::HEALTH_QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); } $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); @@ -545,7 +599,7 @@ App::get('/v1/health/queue/messaging') $size = $client->getQueueSize(); if ($size >= $threshold) { - throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); + throw new Exception(Exception::HEALTH_QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); } $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); @@ -572,7 +626,7 @@ App::get('/v1/health/queue/migrations') $size = $client->getQueueSize(); if ($size >= $threshold) { - throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); + throw new Exception(Exception::HEALTH_QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); } $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); @@ -599,7 +653,7 @@ App::get('/v1/health/queue/functions') $size = $client->getQueueSize(); if ($size >= $threshold) { - throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); + throw new Exception(Exception::HEALTH_QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); } $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); @@ -687,6 +741,47 @@ App::get('/v1/health/anti-virus') $response->dynamic(new Document($output), Response::MODEL_HEALTH_ANTIVIRUS); }); +App::get('/v1/health/queue/failed/:name') + ->desc('Get number of failed queue jobs') + ->groups(['api', 'health']) + ->label('scope', 'health.read') + ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) + ->label('sdk.namespace', 'health') + ->label('sdk.method', 'getFailedJobs') + ->param('name', '', new WhiteList([ + Event::DATABASE_QUEUE_NAME, + Event::DELETE_QUEUE_NAME, + Event::AUDITS_QUEUE_NAME, + Event::MAILS_QUEUE_NAME, + Event::FUNCTIONS_QUEUE_NAME, + Event::USAGE_QUEUE_NAME, + Event::WEBHOOK_CLASS_NAME, + Event::CERTIFICATES_QUEUE_NAME, + Event::BUILDS_QUEUE_NAME, + Event::MESSAGING_QUEUE_NAME, + Event::MIGRATIONS_QUEUE_NAME, + Event::HAMSTER_CLASS_NAME + ]), 'The name of the queue') + ->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true) + ->label('sdk.description', '/docs/references/health/get-failed-queue-jobs.md') + ->label('sdk.response.code', Response::STATUS_CODE_OK) + ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) + ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE) + ->inject('response') + ->inject('queue') + ->action(function (string $name, int|string $threshold, Response $response, Connection $queue) { + $threshold = \intval($threshold); + + $client = new Client($name, $queue); + $failed = $client->countFailedJobs(); + + if ($failed >= $threshold) { + throw new Exception(Exception::HEALTH_QUEUE_SIZE_EXCEEDED, "Queue failed jobs threshold hit. Current size is {$failed} and threshold is {$threshold}."); + } + + $response->dynamic(new Document([ 'size' => $failed ]), Response::MODEL_HEALTH_QUEUE); + }); + App::get('/v1/health/stats') // Currently only used internally ->desc('Get system stats') ->groups(['api', 'health']) diff --git a/app/controllers/api/proxy.php b/app/controllers/api/proxy.php index 3081b3def..8d3f559d6 100644 --- a/app/controllers/api/proxy.php +++ b/app/controllers/api/proxy.php @@ -14,6 +14,7 @@ use Utopia\Database\Helpers\ID; use Utopia\Database\Query; use Utopia\Database\Validator\UID; use Utopia\Domains\Domain; +use Utopia\Logger\Log; use Utopia\Validator\Domain as ValidatorDomain; use Utopia\Validator\Text; use Utopia\Validator\WhiteList; @@ -278,7 +279,8 @@ App::patch('/v1/proxy/rules/:ruleId/verification') ->inject('queueForEvents') ->inject('project') ->inject('dbForConsole') - ->action(function (string $ruleId, Response $response, Certificate $queueForCertificates, Event $queueForEvents, Document $project, Database $dbForConsole) { + ->inject('log') + ->action(function (string $ruleId, Response $response, Certificate $queueForCertificates, Event $queueForEvents, Document $project, Database $dbForConsole, Log $log) { $rule = $dbForConsole->getDocument('rules', $ruleId); if ($rule->isEmpty() || $rule->getAttribute('projectInternalId') !== $project->getInternalId()) { @@ -298,7 +300,14 @@ App::patch('/v1/proxy/rules/:ruleId/verification') $validator = new CNAME($target->get()); // Verify Domain with DNS records $domain = new Domain($rule->getAttribute('domain', '')); + $validationStart = \microtime(true); if (!$validator->isValid($domain->get())) { + $log->addExtra('dnsTiming', \strval(\microtime(true) - $validationStart)); + $log->addTag('dnsDomain', $domain->get()); + + $error = $validator->getLogs(); + $log->addExtra('dnsResponse', \is_array($error) ? \json_encode($error) : \strval($error)); + throw new Exception(Exception::RULE_VERIFICATION_FAILED); } diff --git a/app/controllers/api/teams.php b/app/controllers/api/teams.php index 685c230f4..b8d0a8229 100644 --- a/app/controllers/api/teams.php +++ b/app/controllers/api/teams.php @@ -534,8 +534,8 @@ App::post('/v1/teams/:teamId/memberships') } catch (Duplicate $th) { throw new Exception(Exception::TEAM_INVITE_ALREADY_EXISTS); } - $team->setAttribute('total', $team->getAttribute('total', 0) + 1); - $team = Authorization::skip(fn() => $dbForProject->updateDocument('teams', $team->getId(), $team)); + + Authorization::skip(fn() => $dbForProject->increaseDocumentAttribute('teams', $team->getId(), 'total', 1)); $dbForProject->purgeCachedDocument('users', $invitee->getId()); } else { diff --git a/app/controllers/general.php b/app/controllers/general.php index 2a139622a..0a776c71c 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -617,66 +617,58 @@ App::error() ->inject('response') ->inject('project') ->inject('logger') - ->inject('loggerBreadcrumbs') - ->action(function (Throwable $error, App $utopia, Request $request, Response $response, Document $project, ?Logger $logger, array $loggerBreadcrumbs) { - + ->inject('log') + ->action(function (Throwable $error, App $utopia, Request $request, Response $response, Document $project, ?Logger $logger, Log $log) { $version = App::getEnv('_APP_VERSION', 'UNKNOWN'); $route = $utopia->getRoute(); - $publish = true; if ($error instanceof AppwriteException) { $publish = $error->isPublishable(); + } else { + $publish = $error->getCode() === 0 || $error->getCode() >= 500; } - if ($logger && $publish) { - if ($error->getCode() >= 500 || $error->getCode() === 0) { - try { - /** @var Utopia\Database\Document $user */ - $user = $utopia->getResource('user'); - } catch (\Throwable $th) { - // All good, user is optional information for logger - } - - $log = new Utopia\Logger\Log(); - - if (isset($user) && !$user->isEmpty()) { - $log->setUser(new User($user->getId())); - } - - $log->setNamespace("http"); - $log->setServer(\gethostname()); - $log->setVersion($version); - $log->setType(Log::TYPE_ERROR); - $log->setMessage($error->getMessage()); - - $log->addTag('database', $project->getAttribute('database', 'console')); - $log->addTag('method', $route->getMethod()); - $log->addTag('url', $route->getPath()); - $log->addTag('verboseType', get_class($error)); - $log->addTag('code', $error->getCode()); - $log->addTag('projectId', $project->getId()); - $log->addTag('hostname', $request->getHostname()); - $log->addTag('locale', (string)$request->getParam('locale', $request->getHeader('x-appwrite-locale', ''))); - - $log->addExtra('file', $error->getFile()); - $log->addExtra('line', $error->getLine()); - $log->addExtra('trace', $error->getTraceAsString()); - $log->addExtra('detailedTrace', $error->getTrace()); - $log->addExtra('roles', Authorization::getRoles()); - - $action = $route->getLabel("sdk.namespace", "UNKNOWN_NAMESPACE") . '.' . $route->getLabel("sdk.method", "UNKNOWN_METHOD"); - $log->setAction($action); - - $isProduction = App::getEnv('_APP_ENV', 'development') === 'production'; - $log->setEnvironment($isProduction ? Log::ENVIRONMENT_PRODUCTION : Log::ENVIRONMENT_STAGING); - - foreach ($loggerBreadcrumbs as $loggerBreadcrumb) { - $log->addBreadcrumb($loggerBreadcrumb); - } - - $responseCode = $logger->addLog($log); - Console::info('Log pushed with status code: ' . $responseCode); + if ($logger && ($publish || $error->getCode() === 0)) { + try { + /** @var Utopia\Database\Document $user */ + $user = $utopia->getResource('user'); + } catch (\Throwable $th) { + // All good, user is optional information for logger } + + if (isset($user) && !$user->isEmpty()) { + $log->setUser(new User($user->getId())); + } + + $log->setNamespace("http"); + $log->setServer(\gethostname()); + $log->setVersion($version); + $log->setType(Log::TYPE_ERROR); + $log->setMessage($error->getMessage()); + + $log->addTag('database', $project->getAttribute('database', 'console')); + $log->addTag('method', $route->getMethod()); + $log->addTag('url', $route->getPath()); + $log->addTag('verboseType', get_class($error)); + $log->addTag('code', $error->getCode()); + $log->addTag('projectId', $project->getId()); + $log->addTag('hostname', $request->getHostname()); + $log->addTag('locale', (string)$request->getParam('locale', $request->getHeader('x-appwrite-locale', ''))); + + $log->addExtra('file', $error->getFile()); + $log->addExtra('line', $error->getLine()); + $log->addExtra('trace', $error->getTraceAsString()); + $log->addExtra('detailedTrace', $error->getTrace()); + $log->addExtra('roles', Authorization::getRoles()); + + $action = $route->getLabel("sdk.namespace", "UNKNOWN_NAMESPACE") . '.' . $route->getLabel("sdk.method", "UNKNOWN_METHOD"); + $log->setAction($action); + + $isProduction = App::getEnv('_APP_ENV', 'development') === 'production'; + $log->setEnvironment($isProduction ? Log::ENVIRONMENT_PRODUCTION : Log::ENVIRONMENT_STAGING); + + $responseCode = $logger->addLog($log); + Console::info('Log pushed with status code: ' . $responseCode); } $code = $error->getCode(); diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 85a74ecca..30c0a2ee0 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -179,6 +179,7 @@ App::init() $end = $request->getContentRangeEnd(); $timeLimit = new TimeLimit($abuseKey, $route->getLabel('abuse-limit', 0), $route->getLabel('abuse-time', 3600), $dbForProject); $timeLimit + ->setParam('{projectId}', $project->getId()) ->setParam('{userId}', $user->getId()) ->setParam('{userAgent}', $request->getUserAgent('')) ->setParam('{ip}', $request->getIP()) @@ -350,7 +351,7 @@ App::init() break; case 'magic-url': - if ($project->getAttribute('usersAuthMagicURL', true) === false) { + if (($auths['usersAuthMagicURL'] ?? true) === false) { throw new Exception(Exception::USER_AUTH_METHOD_UNSUPPORTED, 'Magic URL authentication is disabled for this project'); } break; @@ -361,6 +362,12 @@ App::init() } break; + case 'phone': + if (($auths['phone'] ?? true) === false) { + throw new Exception(Exception::USER_AUTH_METHOD_UNSUPPORTED, 'Phone authentication is disabled for this project'); + } + break; + case 'invites': if (($auths['invites'] ?? true) === false) { throw new Exception(Exception::USER_AUTH_METHOD_UNSUPPORTED, 'Invites authentication is disabled for this project'); @@ -386,7 +393,7 @@ App::init() break; default: - throw new Exception(Exception::USER_AUTH_METHOD_UNSUPPORTED, 'Unsupported authentication route'); + throw new Exception(Exception::USER_AUTH_METHOD_UNSUPPORTED, 'Unsupported authentication type: ' . $route->getLabel('auth.type', '')); break; } }); diff --git a/app/controllers/shared/api/auth.php b/app/controllers/shared/api/auth.php index 0b24f8b3a..8698e19ae 100644 --- a/app/controllers/shared/api/auth.php +++ b/app/controllers/shared/api/auth.php @@ -43,7 +43,7 @@ App::init() break; case 'magic-url': - if ($project->getAttribute('usersAuthMagicURL', true) === false) { + if (($auths['usersAuthMagicURL'] ?? true) === false) { throw new Exception(Exception::USER_AUTH_METHOD_UNSUPPORTED, 'Magic URL authentication is disabled for this project'); } break; @@ -54,6 +54,12 @@ App::init() } break; + case 'phone': + if (($auths['phone'] ?? true) === false) { + throw new Exception(Exception::USER_AUTH_METHOD_UNSUPPORTED, 'Phone authentication is disabled for this project'); + } + break; + case 'invites': if (($auths['invites'] ?? true) === false) { throw new Exception(Exception::USER_AUTH_METHOD_UNSUPPORTED, 'Invites authentication is disabled for this project'); diff --git a/app/http.php b/app/http.php index 0b17ae5b8..41759e9f9 100644 --- a/app/http.php +++ b/app/http.php @@ -264,10 +264,9 @@ $http->on('request', function (SwooleRequest $swooleRequest, SwooleResponse $swo // All good, user is optional information for logger } - $loggerBreadcrumbs = $app->getResource("loggerBreadcrumbs"); $route = $app->getRoute(); - $log = new Utopia\Logger\Log(); + $log = $app->getResource("log"); if (isset($user) && !$user->isEmpty()) { $log->setUser(new User($user->getId())); @@ -299,10 +298,6 @@ $http->on('request', function (SwooleRequest $swooleRequest, SwooleResponse $swo $isProduction = App::getEnv('_APP_ENV', 'development') === 'production'; $log->setEnvironment($isProduction ? Log::ENVIRONMENT_PRODUCTION : Log::ENVIRONMENT_STAGING); - foreach ($loggerBreadcrumbs as $loggerBreadcrumb) { - $log->addBreadcrumb($loggerBreadcrumb); - } - $responseCode = $logger->addLog($log); Console::info('Log pushed with status code: ' . $responseCode); } diff --git a/app/init.php b/app/init.php index 26f9c7c34..266c080c9 100644 --- a/app/init.php +++ b/app/init.php @@ -71,6 +71,7 @@ use Appwrite\Hooks\Hooks; use MaxMind\Db\Reader; use PHPMailer\PHPMailer\PHPMailer; use Swoole\Database\PDOProxy; +use Utopia\Logger\Log; use Utopia\Queue; use Utopia\Queue\Connection; use Utopia\Storage\Storage; @@ -201,6 +202,7 @@ const MESSAGE_TYPE_PUSH = 'push'; // Usage metrics const METRIC_TEAMS = 'teams'; const METRIC_USERS = 'users'; +const METRIC_MESSAGES = 'messages'; const METRIC_SESSIONS = 'sessions'; const METRIC_DATABASES = 'databases'; const METRIC_COLLECTIONS = 'collections'; @@ -986,6 +988,7 @@ foreach ($locales as $locale) { ]); // Runtime Execution +App::setResource('log', fn() => new Log()); App::setResource('logger', function ($register) { return $register->get('logger'); }, ['register']); @@ -994,10 +997,6 @@ App::setResource('hooks', function ($register) { return $register->get('hooks'); }, ['register']); -App::setResource('loggerBreadcrumbs', function () { - return []; -}); - App::setResource('register', fn() => $register); App::setResource('locale', fn() => new Locale(App::getEnv('_APP_LOCALE', 'en'))); diff --git a/app/views/install/compose.phtml b/app/views/install/compose.phtml index 87dcaade6..9c66e7282 100644 --- a/app/views/install/compose.phtml +++ b/app/views/install/compose.phtml @@ -143,6 +143,7 @@ services: - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG - _APP_MAINTENANCE_INTERVAL + - _APP_MAINTENANCE_DELAY - _APP_MAINTENANCE_RETENTION_EXECUTION - _APP_MAINTENANCE_RETENTION_CACHE - _APP_MAINTENANCE_RETENTION_ABUSE diff --git a/bin/queue-count-failed b/bin/queue-count-failed new file mode 100644 index 000000000..ca8f2b429 --- /dev/null +++ b/bin/queue-count-failed @@ -0,0 +1,3 @@ +#!/bin/sh + +php /usr/src/code/app/cli.php queue-count --type=failed $@ \ No newline at end of file diff --git a/bin/queue-count-processing b/bin/queue-count-processing new file mode 100644 index 000000000..325d86111 --- /dev/null +++ b/bin/queue-count-processing @@ -0,0 +1,3 @@ +#!/bin/sh + +php /usr/src/code/app/cli.php queue-count --type=processing $@ \ No newline at end of file diff --git a/bin/queue-count-success b/bin/queue-count-success new file mode 100644 index 000000000..34fc54b4c --- /dev/null +++ b/bin/queue-count-success @@ -0,0 +1,3 @@ +#!/bin/sh + +php /usr/src/code/app/cli.php queue-count --type=success $@ \ No newline at end of file diff --git a/bin/queue-retry b/bin/queue-retry new file mode 100644 index 000000000..f9473e6b0 --- /dev/null +++ b/bin/queue-retry @@ -0,0 +1,3 @@ +#!/bin/sh + +php /usr/src/code/app/cli.php queue-retry $@ \ No newline at end of file diff --git a/composer.json b/composer.json index 2c78bcd4a..3e74c0ff5 100644 --- a/composer.json +++ b/composer.json @@ -62,7 +62,7 @@ "utopia-php/platform": "0.5.*", "utopia-php/pools": "0.4.*", "utopia-php/preloader": "0.2.*", - "utopia-php/queue": "0.6.*", + "utopia-php/queue": "0.7.*", "utopia-php/registry": "0.5.*", "utopia-php/storage": "0.18.*", "utopia-php/swoole": "0.8.*", @@ -75,7 +75,7 @@ "adhocore/jwt": "1.1.2", "spomky-labs/otphp": "^10.0", "webonyx/graphql-php": "14.11.*", - "league/csv": "^9.14" + "league/csv": "9.14.*" }, "repositories": [ { diff --git a/composer.lock b/composer.lock index 226d55e38..053690da4 100644 --- a/composer.lock +++ b/composer.lock @@ -1029,16 +1029,16 @@ }, { "name": "symfony/polyfill-php80", - "version": "v1.28.0", + "version": "v1.29.0", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-php80.git", - "reference": "6caa57379c4aec19c0a12a38b59b26487dcfe4b5" + "reference": "87b68208d5c1188808dd7839ee1e6c8ec3b02f1b" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-php80/zipball/6caa57379c4aec19c0a12a38b59b26487dcfe4b5", - "reference": "6caa57379c4aec19c0a12a38b59b26487dcfe4b5", + "url": "https://api.github.com/repos/symfony/polyfill-php80/zipball/87b68208d5c1188808dd7839ee1e6c8ec3b02f1b", + "reference": "87b68208d5c1188808dd7839ee1e6c8ec3b02f1b", "shasum": "" }, "require": { @@ -1046,9 +1046,6 @@ }, "type": "library", "extra": { - "branch-alias": { - "dev-main": "1.28-dev" - }, "thanks": { "name": "symfony/polyfill", "url": "https://github.com/symfony/polyfill" @@ -1092,7 +1089,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-php80/tree/v1.28.0" + "source": "https://github.com/symfony/polyfill-php80/tree/v1.29.0" }, "funding": [ { @@ -1108,7 +1105,7 @@ "type": "tidelift" } ], - "time": "2023-01-26T09:26:14+00:00" + "time": "2024-01-29T20:11:03+00:00" }, { "name": "thecodingmachine/safe", @@ -2264,16 +2261,16 @@ }, { "name": "utopia-php/queue", - "version": "0.6.0", + "version": "0.7.0", "source": { "type": "git", "url": "https://github.com/utopia-php/queue.git", - "reference": "0120bd21904cb2bee34e4571b1737589ffff0eb1" + "reference": "917565256eb94bcab7246f7a746b1a486813761b" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/queue/zipball/0120bd21904cb2bee34e4571b1737589ffff0eb1", - "reference": "0120bd21904cb2bee34e4571b1737589ffff0eb1", + "url": "https://api.github.com/repos/utopia-php/queue/zipball/917565256eb94bcab7246f7a746b1a486813761b", + "reference": "917565256eb94bcab7246f7a746b1a486813761b", "shasum": "" }, "require": { @@ -2319,9 +2316,9 @@ ], "support": { "issues": "https://github.com/utopia-php/queue/issues", - "source": "https://github.com/utopia-php/queue/tree/0.6.0" + "source": "https://github.com/utopia-php/queue/tree/0.7.0" }, - "time": "2023-10-16T16:59:45+00:00" + "time": "2024-01-17T19:00:43+00:00" }, { "name": "utopia-php/registry", @@ -5123,16 +5120,16 @@ }, { "name": "symfony/polyfill-ctype", - "version": "v1.28.0", + "version": "v1.29.0", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-ctype.git", - "reference": "ea208ce43cbb04af6867b4fdddb1bdbf84cc28cb" + "reference": "ef4d7e442ca910c4764bce785146269b30cb5fc4" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-ctype/zipball/ea208ce43cbb04af6867b4fdddb1bdbf84cc28cb", - "reference": "ea208ce43cbb04af6867b4fdddb1bdbf84cc28cb", + "url": "https://api.github.com/repos/symfony/polyfill-ctype/zipball/ef4d7e442ca910c4764bce785146269b30cb5fc4", + "reference": "ef4d7e442ca910c4764bce785146269b30cb5fc4", "shasum": "" }, "require": { @@ -5146,9 +5143,6 @@ }, "type": "library", "extra": { - "branch-alias": { - "dev-main": "1.28-dev" - }, "thanks": { "name": "symfony/polyfill", "url": "https://github.com/symfony/polyfill" @@ -5185,7 +5179,7 @@ "portable" ], "support": { - "source": "https://github.com/symfony/polyfill-ctype/tree/v1.28.0" + "source": "https://github.com/symfony/polyfill-ctype/tree/v1.29.0" }, "funding": [ { @@ -5201,20 +5195,20 @@ "type": "tidelift" } ], - "time": "2023-01-26T09:26:14+00:00" + "time": "2024-01-29T20:11:03+00:00" }, { "name": "symfony/polyfill-mbstring", - "version": "v1.28.0", + "version": "v1.29.0", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-mbstring.git", - "reference": "42292d99c55abe617799667f454222c54c60e229" + "reference": "9773676c8a1bb1f8d4340a62efe641cf76eda7ec" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-mbstring/zipball/42292d99c55abe617799667f454222c54c60e229", - "reference": "42292d99c55abe617799667f454222c54c60e229", + "url": "https://api.github.com/repos/symfony/polyfill-mbstring/zipball/9773676c8a1bb1f8d4340a62efe641cf76eda7ec", + "reference": "9773676c8a1bb1f8d4340a62efe641cf76eda7ec", "shasum": "" }, "require": { @@ -5228,9 +5222,6 @@ }, "type": "library", "extra": { - "branch-alias": { - "dev-main": "1.28-dev" - }, "thanks": { "name": "symfony/polyfill", "url": "https://github.com/symfony/polyfill" @@ -5268,7 +5259,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-mbstring/tree/v1.28.0" + "source": "https://github.com/symfony/polyfill-mbstring/tree/v1.29.0" }, "funding": [ { @@ -5284,7 +5275,7 @@ "type": "tidelift" } ], - "time": "2023-07-28T09:04:16+00:00" + "time": "2024-01-29T20:11:03+00:00" }, { "name": "textalk/websocket", diff --git a/docker-compose.yml b/docker-compose.yml index 72d81cc8a..0a806929d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -582,6 +582,7 @@ services: - _APP_REDIS_PORT - _APP_REDIS_USER - _APP_REDIS_PASS + - _APP_SMS_PROJECTS_DENY_LIST - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA @@ -662,6 +663,7 @@ services: - _APP_MAINTENANCE_RETENTION_AUDIT - _APP_MAINTENANCE_RETENTION_USAGE_HOURLY - _APP_MAINTENANCE_RETENTION_SCHEDULES + - _APP_MAINTENANCE_DELAY appwrite-worker-usage: entrypoint: worker-usage diff --git a/docs/references/health/get-certificate.md b/docs/references/health/get-certificate.md new file mode 100644 index 000000000..bf1eeb838 --- /dev/null +++ b/docs/references/health/get-certificate.md @@ -0,0 +1 @@ +Get the SSL certificate for a domain \ No newline at end of file diff --git a/docs/references/health/get-failed-queue-jobs.md b/docs/references/health/get-failed-queue-jobs.md new file mode 100644 index 000000000..79d4af649 --- /dev/null +++ b/docs/references/health/get-failed-queue-jobs.md @@ -0,0 +1 @@ +Returns the amount of failed jobs in a given queue. diff --git a/src/Appwrite/Extend/Exception.php b/src/Appwrite/Extend/Exception.php index bd1397bc2..29eba3785 100644 --- a/src/Appwrite/Extend/Exception.php +++ b/src/Appwrite/Extend/Exception.php @@ -244,7 +244,9 @@ class Exception extends \Exception public const REALTIME_POLICY_VIOLATION = 'realtime_policy_violation'; /** Health */ - public const QUEUE_SIZE_EXCEEDED = 'queue_size_exceeded'; + public const HEALTH_QUEUE_SIZE_EXCEEDED = 'health_queue_size_exceeded'; + public const HEALTH_CERTIFICATE_EXPIRED = 'health_certificate_expired'; + public const HEALTH_INVALID_HOST = 'health_invalid_host'; /** Provider */ public const PROVIDER_NOT_FOUND = 'provider_not_found'; @@ -276,21 +278,16 @@ class Exception extends \Exception protected string $type = ''; protected array $errors = []; - protected bool $publish = true; + protected bool $publish; public function __construct(string $type = Exception::GENERAL_UNKNOWN, string $message = null, int $code = null, \Throwable $previous = null) { $this->errors = Config::getParam('errors'); $this->type = $type; + $this->code = $code ?? $this->errors[$type]['code']; + $this->message = $message ?? $this->errors[$type]['description']; - if (isset($this->errors[$type])) { - $this->code = $this->errors[$type]['code']; - $this->message = $this->errors[$type]['description']; - $this->publish = $this->errors[$type]['publish'] ?? true; - } - - $this->message = $message ?? $this->message; - $this->code = $code ?? $this->code; + $this->publish = $this->errors[$type]['publish'] ?? ($this->code >= 500); parent::__construct($this->message, $this->code, $previous); } diff --git a/src/Appwrite/Network/Validator/CNAME.php b/src/Appwrite/Network/Validator/CNAME.php index 678a57cec..e1ae061c8 100644 --- a/src/Appwrite/Network/Validator/CNAME.php +++ b/src/Appwrite/Network/Validator/CNAME.php @@ -6,6 +6,11 @@ use Utopia\Validator; class CNAME extends Validator { + /** + * @var mixed + */ + protected mixed $logs; + /** * @var string */ @@ -27,6 +32,14 @@ class CNAME extends Validator return 'Invalid CNAME record'; } + /** + * @return mixed + */ + public function getLogs(): mixed + { + return $this->logs; + } + /** * Check if CNAME record target value matches selected target * @@ -42,6 +55,7 @@ class CNAME extends Validator try { $records = \dns_get_record($domain, DNS_CNAME); + $this->logs = $records; } catch (\Throwable $th) { return false; } diff --git a/src/Appwrite/Platform/Services/Tasks.php b/src/Appwrite/Platform/Services/Tasks.php index 8a7382575..671120a81 100644 --- a/src/Appwrite/Platform/Services/Tasks.php +++ b/src/Appwrite/Platform/Services/Tasks.php @@ -3,6 +3,7 @@ namespace Appwrite\Platform\Services; use Appwrite\Platform\Tasks\CalcTierStats; +use Appwrite\Platform\Tasks\CreateInfMetric; use Appwrite\Platform\Tasks\DeleteOrphanedProjects; use Appwrite\Platform\Tasks\DevGenerateTranslations; use Appwrite\Platform\Tasks\Doctor; @@ -12,6 +13,8 @@ use Appwrite\Platform\Tasks\Install; use Appwrite\Platform\Tasks\Maintenance; use Appwrite\Platform\Tasks\Migrate; use Appwrite\Platform\Tasks\PatchRecreateRepositoriesDocuments; +use Appwrite\Platform\Tasks\QueueCount; +use Appwrite\Platform\Tasks\QueueRetry; use Appwrite\Platform\Tasks\SDKs; use Appwrite\Platform\Tasks\SSL; use Appwrite\Platform\Tasks\ScheduleFunctions; @@ -21,7 +24,6 @@ use Appwrite\Platform\Tasks\Upgrade; use Appwrite\Platform\Tasks\Vars; use Appwrite\Platform\Tasks\Version; use Appwrite\Platform\Tasks\VolumeSync; -use Appwrite\Platform\Tasks\CreateInfMetric; use Utopia\Platform\Service; class Tasks extends Service @@ -30,19 +32,8 @@ class Tasks extends Service { $this->type = self::TYPE_CLI; $this - ->addAction(Version::getName(), new Version()) - ->addAction(Vars::getName(), new Vars()) - ->addAction(SSL::getName(), new SSL()) - ->addAction(Hamster::getName(), new Hamster()) - ->addAction(Doctor::getName(), new Doctor()) - ->addAction(Install::getName(), new Install()) - ->addAction(Upgrade::getName(), new Upgrade()) - ->addAction(Maintenance::getName(), new Maintenance()) - ->addAction(Migrate::getName(), new Migrate()) - ->addAction(SDKs::getName(), new SDKs()) - ->addAction(VolumeSync::getName(), new VolumeSync()) - ->addAction(Specs::getName(), new Specs()) ->addAction(CalcTierStats::getName(), new CalcTierStats()) + ->addAction(CreateInfMetric::getName(), new CreateInfMetric()) ->addAction(DeleteOrphanedProjects::getName(), new DeleteOrphanedProjects()) ->addAction(DevGenerateTranslations::getName(), new DevGenerateTranslations()) ->addAction(Doctor::getName(), new Doctor()) @@ -51,7 +42,10 @@ class Tasks extends Service ->addAction(Install::getName(), new Install()) ->addAction(Maintenance::getName(), new Maintenance()) ->addAction(Migrate::getName(), new Migrate()) + ->addAction(Migrate::getName(), new Migrate()) ->addAction(PatchRecreateRepositoriesDocuments::getName(), new PatchRecreateRepositoriesDocuments()) + ->addAction(QueueCount::getName(), new QueueCount()) + ->addAction(QueueRetry::getName(), new QueueRetry()) ->addAction(SDKs::getName(), new SDKs()) ->addAction(SSL::getName(), new SSL()) ->addAction(ScheduleFunctions::getName(), new ScheduleFunctions()) @@ -61,7 +55,6 @@ class Tasks extends Service ->addAction(Vars::getName(), new Vars()) ->addAction(Version::getName(), new Version()) ->addAction(VolumeSync::getName(), new VolumeSync()) - ->addAction(CreateInfMetric::getName(), new CreateInfMetric()) ; } } diff --git a/src/Appwrite/Platform/Tasks/Maintenance.php b/src/Appwrite/Platform/Tasks/Maintenance.php index e46de1b52..3245a9244 100644 --- a/src/Appwrite/Platform/Tasks/Maintenance.php +++ b/src/Appwrite/Platform/Tasks/Maintenance.php @@ -36,6 +36,7 @@ class Maintenance extends Action // # of days in seconds (1 day = 86400s) $interval = (int) App::getEnv('_APP_MAINTENANCE_INTERVAL', '86400'); + $delay = (int) App::getEnv('_APP_MAINTENANCE_DELAY', '0'); $usageStatsRetentionHourly = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_USAGE_HOURLY', '8640000'); //100 days $cacheRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_CACHE', '2592000'); // 30 days $schedulesDeletionRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_SCHEDULES', '86400'); // 1 Day @@ -60,7 +61,7 @@ class Maintenance extends Action $this->notifyDeleteCache($cacheRetention, $queueForDeletes); $this->notifyDeleteSchedules($schedulesDeletionRetention, $queueForDeletes); $this->notifyDeleteTargets($queueForDeletes); - }, $interval); + }, $interval, $delay); } protected function foreachProject(Database $dbForConsole, callable $callback): void diff --git a/src/Appwrite/Platform/Tasks/QueueCount.php b/src/Appwrite/Platform/Tasks/QueueCount.php new file mode 100644 index 000000000..9b7bba82d --- /dev/null +++ b/src/Appwrite/Platform/Tasks/QueueCount.php @@ -0,0 +1,70 @@ +desc('Return the number of from a specific queue identified by the name parameter with a specific type') + ->param('name', '', new WhiteList([ + Event::DATABASE_QUEUE_NAME, + Event::DELETE_QUEUE_NAME, + Event::AUDITS_QUEUE_NAME, + Event::MAILS_QUEUE_NAME, + Event::FUNCTIONS_QUEUE_NAME, + Event::USAGE_QUEUE_NAME, + Event::WEBHOOK_QUEUE_NAME, + Event::CERTIFICATES_QUEUE_NAME, + Event::BUILDS_QUEUE_NAME, + Event::MESSAGING_QUEUE_NAME, + Event::MIGRATIONS_QUEUE_NAME, + Event::HAMSTER_QUEUE_NAME + ]), 'Queue name') + ->param('type', '', new WhiteList([ + 'success', + 'failed', + 'processing', + ]), 'Queue type') + ->inject('queue') + ->callback(fn ($name, $type, $queue) => $this->action($name, $type, $queue)); + } + + /** + * @param string $name The name of the queue to count the jobs from + * @param string $type The type of jobs to count + * @param Connection $queue + */ + public function action(string $name, string $type, Connection $queue): void + { + if (!$name) { + Console::error('Missing required parameter $name'); + return; + } + + $queueClient = new Client($name, $queue); + + $count = match ($type) { + 'success' => $queueClient->countSuccessfulJobs(), + 'failed' => $queueClient->countFailedJobs(), + 'processing' => $queueClient->countProcessingJobs(), + default => 0 + }; + + Console::log("Queue: '{$name}' has {$count} {$type} jobs."); + } +} diff --git a/src/Appwrite/Platform/Tasks/QueueRetry.php b/src/Appwrite/Platform/Tasks/QueueRetry.php new file mode 100644 index 000000000..cba68f993 --- /dev/null +++ b/src/Appwrite/Platform/Tasks/QueueRetry.php @@ -0,0 +1,64 @@ +desc('Retry failed jobs from a specific queue identified by the name parameter') + ->param('name', '', new WhiteList([ + Event::DATABASE_QUEUE_NAME, + Event::DELETE_QUEUE_NAME, + Event::AUDITS_QUEUE_NAME, + Event::MAILS_QUEUE_NAME, + Event::FUNCTIONS_QUEUE_NAME, + Event::USAGE_QUEUE_NAME, + Event::WEBHOOK_CLASS_NAME, + Event::CERTIFICATES_QUEUE_NAME, + Event::BUILDS_QUEUE_NAME, + Event::MESSAGING_QUEUE_NAME, + Event::MIGRATIONS_QUEUE_NAME, + Event::HAMSTER_CLASS_NAME + ]), 'Queue name') + ->inject('queue') + ->callback(fn ($name, $queue) => $this->action($name, $queue)); + } + + /** + * @param string $name The name of the queue to retry jobs from + * @param Connection $queue + */ + public function action(string $name, Connection $queue): void + { + if (!$name) { + Console::error('Missing required parameter $name'); + return; + } + + $queueClient = new Client($name, $queue); + + if ($queueClient->countFailedJobs() === 0) { + Console::error('No failed jobs found.'); + return; + } + + Console::log('Retrying failed jobs...'); + + $queueClient->retry(); + } +} diff --git a/src/Appwrite/Platform/Tasks/SSL.php b/src/Appwrite/Platform/Tasks/SSL.php index 6dbf4dcd7..12cb0d6be 100644 --- a/src/Appwrite/Platform/Tasks/SSL.php +++ b/src/Appwrite/Platform/Tasks/SSL.php @@ -7,6 +7,7 @@ use Appwrite\Event\Certificate; use Utopia\App; use Utopia\CLI\Console; use Utopia\Database\Document; +use Utopia\Validator\Boolean; use Utopia\Validator\Hostname; class SSL extends Action @@ -21,19 +22,22 @@ class SSL extends Action $this ->desc('Validate server certificates') ->param('domain', App::getEnv('_APP_DOMAIN', ''), new Hostname(), 'Domain to generate certificate for. If empty, main domain will be used.', true) + ->param('skip-check', true, new Boolean(true), 'If DNS and renew check should be skipped. Defaults to true, and when true, all jobs will result in certificate generation attempt.', true) ->inject('queueForCertificates') - ->callback(fn (string $domain, Certificate $queueForCertificates) => $this->action($domain, $queueForCertificates)); + ->callback(fn (string $domain, bool|string $skipCheck, Certificate $queueForCertificates) => $this->action($domain, $skipCheck, $queueForCertificates)); } - public function action(string $domain, Certificate $queueForCertificates): void + public function action(string $domain, bool|string $skipCheck, Certificate $queueForCertificates): void { + $skipCheck = \strval($skipCheck) === 'true'; + Console::success('Scheduling a job to issue a TLS certificate for domain: ' . $domain); $queueForCertificates ->setDomain(new Document([ 'domain' => $domain ])) - ->setSkipRenewCheck(true) + ->setSkipRenewCheck($skipCheck) ->trigger(); } } diff --git a/src/Appwrite/Platform/Workers/Certificates.php b/src/Appwrite/Platform/Workers/Certificates.php index b91f886a2..13e85d473 100644 --- a/src/Appwrite/Platform/Workers/Certificates.php +++ b/src/Appwrite/Platform/Workers/Certificates.php @@ -75,7 +75,7 @@ class Certificates extends Action $log->addTag('domain', $domain->get()); - $this->execute($domain, $dbForConsole, $queueForMails, $queueForEvents, $queueForFunctions, $skipRenewCheck); + $this->execute($domain, $dbForConsole, $queueForMails, $queueForEvents, $queueForFunctions, $log, $skipRenewCheck); } /** @@ -89,7 +89,7 @@ class Certificates extends Action * @throws Throwable * @throws \Utopia\Database\Exception */ - private function execute(Domain $domain, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, bool $skipRenewCheck = false): void + private function execute(Domain $domain, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log, bool $skipRenewCheck = false): void { /** * 1. Read arguments and validate domain @@ -143,11 +143,11 @@ class Certificates extends Action if (!$skipRenewCheck) { $mainDomain = $this->getMainDomain(); $isMainDomain = !isset($mainDomain) || $domain->get() === $mainDomain; - $this->validateDomain($domain, $isMainDomain); + $this->validateDomain($domain, $isMainDomain, $log); } // If certificate exists already, double-check expiry date. Skip if job is forced - if (!$skipRenewCheck && !$this->isRenewRequired($domain->get())) { + if (!$skipRenewCheck && !$this->isRenewRequired($domain->get(), $log)) { throw new Exception('Renew isn\'t required.'); } @@ -185,6 +185,8 @@ class Certificates extends Action // Send email to security email $this->notifyError($domain->get(), $e->getMessage(), $attempts, $queueForMails); + + throw $e; } finally { // All actions result in new updatedAt date $certificate->setAttribute('updated', DateTime::now()); @@ -252,7 +254,7 @@ class Certificates extends Action * @return void * @throws Exception */ - private function validateDomain(Domain $domain, bool $isMainDomain): void + private function validateDomain(Domain $domain, bool $isMainDomain, Log $log): void { if (empty($domain->get())) { throw new Exception('Missing certificate domain.'); @@ -272,8 +274,15 @@ class Certificates extends Action } // Verify domain with DNS records + $validationStart = \microtime(true); $validator = new CNAME($target->get()); if (!$validator->isValid($domain->get())) { + $log->addExtra('dnsTiming', \strval(\microtime(true) - $validationStart)); + $log->addTag('dnsDomain', $domain->get()); + + $error = $validator->getLogs(); + $log->addExtra('dnsResponse', \is_array($error) ? \json_encode($error) : \strval($error)); + throw new Exception('Failed to verify domain DNS records.'); } } else { @@ -289,7 +298,7 @@ class Certificates extends Action * @return bool True, if certificate needs to be renewed * @throws Exception */ - private function isRenewRequired(string $domain): bool + private function isRenewRequired(string $domain, Log $log): bool { $certPath = APP_STORAGE_CERTIFICATES . '/' . $domain . '/cert.pem'; if (\file_exists($certPath)) { @@ -299,12 +308,15 @@ class Certificates extends Action $validTo = $certData['validTo_time_t'] ?? 0; if (empty($validTo)) { + $log->addTag('certificateDomain', $domain); throw new Exception('Unable to read certificate file (cert.pem).'); } // LetsEncrypt allows renewal 30 days before expiry $expiryInAdvance = (60 * 60 * 24 * 30); if ($validTo - $expiryInAdvance > \time()) { + $log->addTag('certificateDomain', $domain); + $log->addExtra('certificateData', \is_array($certData) ? \json_encode($certData) : \strval($certData)); return false; } } diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index f22fa9ca4..2875964d5 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -630,12 +630,7 @@ class Deletes extends Action $teamId = $document->getAttribute('teamId'); $team = $dbForProject->getDocument('teams', $teamId); if (!$team->isEmpty()) { - $team = $dbForProject->updateDocument( - 'teams', - $teamId, - // Ensure that total >= 0 - $team->setAttribute('total', \max($team->getAttribute('total', 0) - 1, 0)) - ); + $dbForProject->decreaseDocumentAttribute('teams', $teamId, 'total', 1, 0); } } }); diff --git a/src/Appwrite/Platform/Workers/Messaging.php b/src/Appwrite/Platform/Workers/Messaging.php index 57a0e5f64..e39a2dd82 100644 --- a/src/Appwrite/Platform/Workers/Messaging.php +++ b/src/Appwrite/Platform/Workers/Messaging.php @@ -2,23 +2,22 @@ namespace Appwrite\Platform\Workers; +use Appwrite\Event\Usage; use Appwrite\Extend\Exception; use Appwrite\Messaging\Status as MessageStatus; use Utopia\App; use Utopia\CLI\Console; -use Utopia\Database\Helpers\ID; use Utopia\DSN\DSN; -use Utopia\Logger\Log; -use Utopia\Platform\Action; -use Utopia\Queue\Message; use Utopia\Database\Database; use Utopia\Database\DateTime; use Utopia\Database\Document; +use Utopia\Database\Helpers\ID; use Utopia\Database\Query; +use Utopia\Logger\Log; use Utopia\Messaging\Adapter\Email as EmailAdapter; use Utopia\Messaging\Adapter\Email\Mailgun; -use Utopia\Messaging\Adapter\Email\Sendgrid; use Utopia\Messaging\Adapter\Email\SMTP; +use Utopia\Messaging\Adapter\Email\Sendgrid; use Utopia\Messaging\Adapter\Push as PushAdapter; use Utopia\Messaging\Adapter\Push\APNS; use Utopia\Messaging\Adapter\Push\FCM; @@ -32,7 +31,8 @@ use Utopia\Messaging\Adapter\SMS\Vonage; use Utopia\Messaging\Messages\Email; use Utopia\Messaging\Messages\Push; use Utopia\Messaging\Messages\SMS; -use Utopia\Messaging\Response; +use Utopia\Platform\Action; +use Utopia\Queue\Message; use function Swoole\Coroutine\batch; @@ -53,31 +53,40 @@ class Messaging extends Action ->inject('message') ->inject('log') ->inject('dbForProject') - ->callback(fn(Message $message, Log $log, Database $dbForProject) => $this->action($message, $log, $dbForProject)); + ->inject('queueForUsage') + ->callback(fn(Message $message, Log $log, Database $dbForProject, Usage $queueForUsage) => $this->action($message, $log, $dbForProject, $queueForUsage)); } /** * @param Message $message * @param Log $log * @param Database $dbForProject + * @param Usage $queueForUsage * @return void * @throws Exception */ - public function action(Message $message, Log $log, Database $dbForProject): void + public function action(Message $message, Log $log, Database $dbForProject, Usage $queueForUsage): void { $payload = $message->getPayload() ?? []; if (empty($payload)) { - throw new \Exception('Payload not found.'); + throw new Exception('Missing payload'); } + if ( !\is_null($payload['message']) && !\is_null($payload['recipients']) && $payload['providerType'] === MESSAGE_TYPE_SMS ) { // Message was triggered internally - $this->processInternalSMSMessage($log, new Document($payload['message']), $payload['recipients']); + $this->processInternalSMSMessage( + new Document($payload['message']), + new Document($payload['project'] ?? []), + $payload['recipients'], + $queueForUsage, + $log, + ); } else { $message = $dbForProject->getDocument('messages', $payload['messageId']); @@ -299,12 +308,26 @@ class Messaging extends Action $dbForProject->updateDocument('messages', $message->getId(), $message); } - private function processInternalSMSMessage(Log $log, Document $message, array $recipients): void + private function processInternalSMSMessage(Document $message, Document $project, array $recipients, Usage $queueForUsage, Log $log): void { if (empty(App::getEnv('_APP_SMS_PROVIDER')) || empty(App::getEnv('_APP_SMS_FROM'))) { throw new \Exception('Skipped SMS processing. Missing "_APP_SMS_PROVIDER" or "_APP_SMS_FROM" environment variables.'); } + if ($project->isEmpty()) { + throw new Exception('Project not set in payload'); + } + + Console::log('Project: ' . $project->getId()); + + $denyList = App::getEnv('_APP_SMS_PROJECTS_DENY_LIST', ''); + $denyList = explode(',', $denyList); + + if (\in_array($project->getId(), $denyList)) { + Console::error('Project is in the deny list. Skipping...'); + return; + } + $smsDSN = new DSN(App::getEnv('_APP_SMS_PROVIDER')); $host = $smsDSN->getHost(); $password = $smsDSN->getPassword(); @@ -354,16 +377,21 @@ class Messaging extends Action $batches = \array_chunk($recipients, $maxBatchSize); $batchIndex = 0; - batch(\array_map(function ($batch) use ($message, $provider, $adapter, $batchIndex) { - return function () use ($batch, $message, $provider, $adapter, $batchIndex) { + batch(\array_map(function ($batch) use ($message, $provider, $adapter, $batchIndex, $project, $queueForUsage) { + return function () use ($batch, $message, $provider, $adapter, $batchIndex, $project, $queueForUsage) { $message->setAttribute('to', $batch); $data = $this->buildSMSMessage($message, $provider); try { $adapter->send($data); + + $queueForUsage + ->setProject($project) + ->addMetric(METRIC_MESSAGES, 1) + ->trigger(); } catch (\Exception $e) { - Console::error('Failed sending to targets ' . $batchIndex + 1 . '-' . \count($batch) . ' with error: ' . $e->getMessage()); // TODO: Find a way to log into Sentry + throw new Exception('Failed sending to targets ' . $batchIndex + 1 . '-' . \count($batch) . ' with error: ' . $e->getMessage(), 500); } }; }, $batches)); @@ -376,6 +404,7 @@ class Messaging extends Action private function sms(Document $provider): ?SMSAdapter { $credentials = $provider->getAttribute('credentials'); + return match ($provider->getAttribute('provider')) { 'mock' => new Mock('username', 'password'), 'twilio' => new Twilio($credentials['accountSid'], $credentials['authToken']), @@ -390,6 +419,7 @@ class Messaging extends Action private function push(Document $provider): ?PushAdapter { $credentials = $provider->getAttribute('credentials'); + return match ($provider->getAttribute('provider')) { 'mock' => new Mock('username', 'password'), 'apns' => new APNS( @@ -407,6 +437,7 @@ class Messaging extends Action { $credentials = $provider->getAttribute('credentials', []); $options = $provider->getAttribute('options', []); + return match ($provider->getAttribute('provider')) { 'mock' => new Mock('username', 'password'), 'smtp' => new SMTP( diff --git a/src/Appwrite/Platform/Workers/Usage.php b/src/Appwrite/Platform/Workers/Usage.php index 5537fae50..2097f101b 100644 --- a/src/Appwrite/Platform/Workers/Usage.php +++ b/src/Appwrite/Platform/Workers/Usage.php @@ -20,7 +20,6 @@ class Usage extends Action ]; protected const INFINITY_PERIOD = '_inf_'; - protected const DEBUG_PROJECT_ID = 85293; public static function getName(): string { return 'usage'; @@ -70,17 +69,6 @@ class Usage extends Action getProjectDB: $getProjectDB ); } - if ($project->getInternalId() == self::DEBUG_PROJECT_ID) { - var_dump([ - 'type' => 'payload', - 'project' => $project->getInternalId(), - 'database' => $project['database'] ?? '', - $payload['metrics'] - ]); - - var_dump('=========================='); - } - self::$stats[$projectId]['project'] = $project; foreach ($payload['metrics'] ?? [] as $metric) { if (!isset(self::$stats[$projectId]['keys'][$metric['key']])) { @@ -91,7 +79,6 @@ class Usage extends Action } } - /** * On Documents that tied by relations like functions>deployments>build || documents>collection>database || buckets>files. * When we remove a parent document we need to deduct his children aggregation from the project scope. diff --git a/src/Appwrite/Platform/Workers/UsageHook.php b/src/Appwrite/Platform/Workers/UsageHook.php index 68e6a3fed..4781b1e89 100644 --- a/src/Appwrite/Platform/Workers/UsageHook.php +++ b/src/Appwrite/Platform/Workers/UsageHook.php @@ -57,14 +57,6 @@ class UsageHook extends Usage try { $dbForProject = $getProjectDB($data['project']); - if ($projectInternalId == 85293) { - var_dump([ - 'project' => $projectInternalId, - 'database' => $database, - 'time' => DateTime::now(), - 'data' => $data['keys'] - ]); - } foreach ($data['keys'] ?? [] as $key => $value) { if ($value == 0) { continue; @@ -75,15 +67,6 @@ class UsageHook extends Usage $id = \md5("{$time}_{$period}_{$key}"); try { - if ($projectInternalId == self::DEBUG_PROJECT_ID) { - var_dump([ - 'type' => 'create', - 'period' => $period, - 'metric' => $key, - 'id' => $id, - 'value' => $value - ]); - } $dbForProject->createDocument('stats_v2', new Document([ '$id' => $id, 'period' => $period, @@ -94,15 +77,6 @@ class UsageHook extends Usage ])); } catch (Duplicate $th) { if ($value < 0) { - if ($projectInternalId == self::DEBUG_PROJECT_ID) { - var_dump([ - 'type' => 'decrease', - 'period' => $period, - 'metric' => $key, - 'id' => $id, - 'value' => $value - ]); - } $dbForProject->decreaseDocumentAttribute( 'stats_v2', $id, @@ -110,15 +84,6 @@ class UsageHook extends Usage abs($value) ); } else { - if ($projectInternalId == self::DEBUG_PROJECT_ID) { - var_dump([ - 'type' => 'increase', - 'period' => $period, - 'metric' => $key, - 'id' => $id, - 'value' => $value - ]); - } $dbForProject->increaseDocumentAttribute( 'stats_v2', $id, diff --git a/src/Appwrite/Utopia/Response.php b/src/Appwrite/Utopia/Response.php index 7ee01f53e..1a4bc7fb5 100644 --- a/src/Appwrite/Utopia/Response.php +++ b/src/Appwrite/Utopia/Response.php @@ -70,6 +70,7 @@ use Appwrite\Utopia\Response\Model\Token; use Appwrite\Utopia\Response\Model\Webhook; use Appwrite\Utopia\Response\Model\Preferences; use Appwrite\Utopia\Response\Model\HealthAntivirus; +use Appwrite\Utopia\Response\Model\HealthCertificate; use Appwrite\Utopia\Response\Model\HealthQueue; use Appwrite\Utopia\Response\Model\HealthStatus; use Appwrite\Utopia\Response\Model\HealthTime; @@ -279,6 +280,7 @@ class Response extends SwooleResponse public const MODEL_HEALTH_QUEUE = 'healthQueue'; public const MODEL_HEALTH_TIME = 'healthTime'; public const MODEL_HEALTH_ANTIVIRUS = 'healthAntivirus'; + public const MODEL_HEALTH_CERTIFICATE = 'healthCertificate'; public const MODEL_HEALTH_STATUS_LIST = 'healthStatusList'; // Console @@ -421,6 +423,7 @@ class Response extends SwooleResponse ->setModel(new HealthAntivirus()) ->setModel(new HealthQueue()) ->setModel(new HealthStatus()) + ->setModel(new HealthCertificate()) ->setModel(new HealthTime()) ->setModel(new HealthVersion()) ->setModel(new Metric()) diff --git a/src/Appwrite/Utopia/Response/Model/HealthCertificate.php b/src/Appwrite/Utopia/Response/Model/HealthCertificate.php new file mode 100644 index 000000000..e4990acc0 --- /dev/null +++ b/src/Appwrite/Utopia/Response/Model/HealthCertificate.php @@ -0,0 +1,71 @@ +addRule('name', [ + 'type' => self::TYPE_STRING, + 'description' => 'Certificate name', + 'default' => '', + 'example' => '/CN=www.google.com', + ]) + ->addRule('subjectSN', [ + 'type' => self::TYPE_STRING, + 'description' => 'Subject SN', + 'default' => 'www.google.com', + 'example' => '', + ]) + ->addRule('issuerOrganisation', [ + 'type' => self::TYPE_STRING, + 'description' => 'Issuer organisation', + 'default' => 'Google Trust Services LLC', + 'example' => '', + ]) + ->addRule('validFrom', [ + 'type' => self::TYPE_STRING, + 'description' => 'Valid from', + 'default' => '', + 'example' => '1704200998', + ]) + ->addRule('validTo', [ + 'type' => self::TYPE_STRING, + 'description' => 'Valid to', + 'default' => '', + 'example' => '1711458597', + ]) + ->addRule('signatureTypeSN', [ + 'type' => self::TYPE_STRING, + 'description' => 'Signature type SN', + 'default' => '', + 'example' => 'RSA-SHA256', + ]) + ; + } + + /** + * Get Name + * + * @return string + */ + public function getName(): string + { + return 'Health Certificate'; + } + + /** + * Get Type + * + * @return string + */ + public function getType(): string + { + return Response::MODEL_HEALTH_CERTIFICATE; + } +} diff --git a/tests/e2e/Services/Health/HealthCustomServerTest.php b/tests/e2e/Services/Health/HealthCustomServerTest.php index 3ea1b884a..c817222c4 100644 --- a/tests/e2e/Services/Health/HealthCustomServerTest.php +++ b/tests/e2e/Services/Health/HealthCustomServerTest.php @@ -424,4 +424,74 @@ class HealthCustomServerTest extends Scope return []; } + + public function testCertificateValidity(): array + { + /** + * Test for SUCCESS + */ + $response = $this->client->call(Client::METHOD_GET, '/health/certificate?domain=www.google.com', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), []); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals('/CN=www.google.com', $response['body']['name']); + $this->assertEquals('www.google.com', $response['body']['subjectSN']); + $this->assertEquals('Google Trust Services LLC', $response['body']['issuerOrganisation']); + $this->assertIsInt($response['body']['validFrom']); + $this->assertIsInt($response['body']['validTo']); + + $response = $this->client->call(Client::METHOD_GET, '/health/certificate?domain=appwrite.io', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), []); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals('/CN=appwrite.io', $response['body']['name']); + $this->assertEquals('appwrite.io', $response['body']['subjectSN']); + $this->assertEquals("Let's Encrypt", $response['body']['issuerOrganisation']); + $this->assertIsInt($response['body']['validFrom']); + $this->assertIsInt($response['body']['validTo']); + + $response = $this->client->call(Client::METHOD_GET, '/health/certificate?domain=https://google.com', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), []); + + $this->assertEquals(200, $response['headers']['status-code']); + + /** + * Test for FAILURE + */ + $response = $this->client->call(Client::METHOD_GET, '/health/certificate?domain=localhost', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), []); + + $this->assertEquals(400, $response['headers']['status-code']); + + $response = $this->client->call(Client::METHOD_GET, '/health/certificate?domain=doesnotexist.com', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), []); + + $this->assertEquals(404, $response['headers']['status-code']); + + $response = $this->client->call(Client::METHOD_GET, '/health/certificate?domain=www.google.com/usr/src/local', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), []); + + $this->assertEquals(400, $response['headers']['status-code']); + + $response = $this->client->call(Client::METHOD_GET, '/health/certificate?domain=', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), []); + + $this->assertEquals(400, $response['headers']['status-code']); + + return []; + } }