CodeMirror/appwrite
1
0
Fork 0
mirror of synced 2024-06-27 18:50:47 +12:00
Code Issues Projects Releases Wiki Activity

Merge branch 'refactor-usage-sn' of github.com:appwrite/appwrite into usage-queue-poc

Browse source 
 Conflicts:
	composer.json
	composer.lock
This commit is contained in:
shimon 2024-02-08 09:29:12 +02:00
parent 93a99b561c 243ddd4b33
commit 632f78140f
43 changed files with 699 additions and 605 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.env
View file

@ -58,6 +58,7 @@ _APP_SMTP_USERNAME=
_APP_SMTP_PASSWORD=
_APP_SMS_PROVIDER=sms://username:password@mock
_APP_SMS_FROM=+123456789
_APP_SMS_PROJECTS_DENY_LIST=
_APP_STORAGE_LIMIT=30000000
_APP_STORAGE_PREVIEW_LIMIT=20000000
_APP_FUNCTIONS_SIZE_LIMIT=30000000
@ -72,6 +73,7 @@ _APP_EXECUTOR_SECRET=your-secret-key
_APP_EXECUTOR_HOST=http://proxy/v1
_APP_FUNCTIONS_RUNTIMES=php-8.0,node-18.0,python-3.9,ruby-3.1
_APP_MAINTENANCE_INTERVAL=86400
_APP_MAINTENANCE_DELAY=
_APP_MAINTENANCE_RETENTION_CACHE=2592000
_APP_MAINTENANCE_RETENTION_EXECUTION=1209600
_APP_MAINTENANCE_RETENTION_ABUSE=86400

4
Dockerfile
View file

@ -84,6 +84,10 @@ RUN chmod +x /usr/local/bin/doctor && \
chmod +x /usr/local/bin/ssl && \
chmod +x /usr/local/bin/test && \
chmod +x /usr/local/bin/vars && \
chmod +x /usr/local/bin/queue-retry && \
chmod +x /usr/local/bin/queue-count-failed && \
chmod +x /usr/local/bin/queue-count-processing && \
chmod +x /usr/local/bin/queue-count-success && \
chmod +x /usr/local/bin/worker-audits && \
chmod +x /usr/local/bin/worker-certificates && \
chmod +x /usr/local/bin/worker-databases && \

17
app/config/errors.php
View file

@ -680,6 +680,7 @@ return [
'name' => Exception::RULE_VERIFICATION_FAILED,
'description' => 'Domain verification failed. Please check if your DNS records are correct and try again.',
'code' => 401,
'publish' => true
],
Exception::PROJECT_SMTP_CONFIG_INVALID => [
'name' => Exception::PROJECT_SMTP_CONFIG_INVALID,
@ -767,10 +768,22 @@ return [
],
/** Health */
Exception::QUEUE_SIZE_EXCEEDED => [
'name' => Exception::QUEUE_SIZE_EXCEEDED,
Exception::HEALTH_QUEUE_SIZE_EXCEEDED => [
'name' => Exception::HEALTH_QUEUE_SIZE_EXCEEDED,
'description' => 'Queue size threshold hit.',
'code' => 503,
'publish' => false
],
Exception::HEALTH_CERTIFICATE_EXPIRED => [
'name' => Exception::HEALTH_CERTIFICATE_EXPIRED,
'description' => 'The SSL certificate for the specified domain has expired and is no longer valid.',
'code' => 404,
],
Exception::HEALTH_INVALID_HOST => [
'name' => Exception::HEALTH_INVALID_HOST,
'description' => 'Failed to establish a connection to the specified domain. Please verify the domain name and ensure that the server is running and accessible.',
'code' => 404,
],
];

2
app/config/specs/open-api3-latest-client.json
View file

File diff suppressed because one or more lines are too long

2
app/config/specs/open-api3-latest-console.json
View file

File diff suppressed because one or more lines are too long

2
app/config/specs/open-api3-latest-server.json
View file

File diff suppressed because one or more lines are too long

2
app/config/specs/swagger2-latest-client.json
View file

File diff suppressed because one or more lines are too long

2
app/config/specs/swagger2-latest-console.json
View file

File diff suppressed because one or more lines are too long

2
app/config/specs/swagger2-latest-server.json
View file

File diff suppressed because one or more lines are too long

9
app/config/variables.php
View file

@ -948,6 +948,15 @@ return [
'question' => '',
'filter' => ''
],
[
'name' => '_APP_MAINTENANCE_DELAY',
'description' => 'Delay value containing the number of seconds that the Appwrite maintenance process should wait before executing system cleanups and optimizations. The default value is 0 seconds.',
'introduction' => '1.5.0',
'default' => '0',
'required' => false,
'question' => '',
'filter' => ''
],
[
'name' => '_APP_MAINTENANCE_RETENTION_CACHE',
'description' => 'The maximum duration (in seconds) upto which to retain cached files. The default value is 2592000 seconds (30 days).',

23
app/controllers/api/account.php
View file

@ -889,7 +889,7 @@ App::delete('/v1/account/identities/:identityId')
App::post('/v1/account/sessions/magic-url')
->desc('Create magic URL session')
->groups(['api', 'account'])
->groups(['api', 'account', 'auth'])
->label('scope', 'public')
->label('auth.type', 'magic-url')
->label('audits.event', 'session.create')
@ -902,8 +902,8 @@ App::post('/v1/account/sessions/magic-url')
->label('sdk.response.code', Response::STATUS_CODE_CREATED)
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
->label('sdk.response.model', Response::MODEL_TOKEN)
->label('abuse-limit', 10)
->label('abuse-key', 'url:{url},email:{param-email}')
->label('abuse-limit', 60)
->label('abuse-key', ['url:{url},email:{param-email}', 'url:{url},ip:{ip}'])
->param('userId', '', new CustomId(), 'Unique Id. Choose a custom ID or generate a random ID with `ID.unique()`. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.')
->param('email', '', new Email(), 'User email.')
->param('url', '', fn($clients) => new Host($clients), 'URL to redirect the user back to your app from the magic URL login. Only URLs from hostnames in your project platform list are allowed. This requirement helps to prevent an [open redirect](https://cheatsheetseries.owasp.org/cheatsheets/Unvalidated_Redirects_and_Forwards_Cheat_Sheet.html) attack against your project API.', true, ['clients'])
@ -1223,7 +1223,7 @@ App::put('/v1/account/sessions/magic-url')
App::post('/v1/account/sessions/phone')
->desc('Create phone session')
->groups(['api', 'account'])
->groups(['api', 'account', 'auth'])
->label('scope', 'public')
->label('auth.type', 'phone')
->label('audits.event', 'session.create')
@ -1237,7 +1237,7 @@ App::post('/v1/account/sessions/phone')
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
->label('sdk.response.model', Response::MODEL_TOKEN)
->label('abuse-limit', 10)
->label('abuse-key', 'url:{url},phone:{param-phone}')
->label('abuse-key', ['url:{url},phone:{param-phone}', 'url:{url},ip:{ip}'])
->param('userId', '', new CustomId(), 'Unique Id. Choose a custom ID or generate a random ID with `ID.unique()`. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.')
->param('phone', '', new Phone(), 'Phone number. Format this number with a leading \'+\' and a country code, e.g., +16175551212.')
->inject('request')
@ -1336,9 +1336,12 @@ App::post('/v1/account/sessions/phone')
$message = $message->setParam('{{token}}', $secret);
$message = $message->render();
var_dump($request->getIP());
var_dump($project->getId());
$queueForMessaging
->setRecipient($phone)
->setMessage($message)
->setProject($project)
->trigger();
$queueForEvents->setPayload(
@ -2388,7 +2391,7 @@ App::post('/v1/account/recovery')
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
->label('sdk.response.model', Response::MODEL_TOKEN)
->label('abuse-limit', 10)
->label('abuse-key', ['url:{url},email:{param-email}', 'ip:{ip}'])
->label('abuse-key', ['url:{url},email:{param-email}', 'url:{url},ip:{ip}'])
->param('email', '', new Email(), 'User email.')
->param('url', '', fn ($clients) => new Host($clients), 'URL to redirect the user back to your app from the recovery email. Only URLs from hostnames in your project platform list are allowed. This requirement helps to prevent an [open redirect](https://cheatsheetseries.owasp.org/cheatsheets/Unvalidated_Redirects_and_Forwards_Cheat_Sheet.html) attack against your project API.', false, ['clients'])
->inject('request')
@ -2861,8 +2864,9 @@ App::put('/v1/account/verification')
App::post('/v1/account/verification/phone')
->desc('Create phone verification')
->groups(['api', 'account'])
->groups(['api', 'account', 'auth'])
->label('scope', 'account')
->label('auth.type', 'phone')
->label('event', 'users.[userId].verification.[tokenId].create')
->label('audits.event', 'verification.create')
->label('audits.resource', 'user/{response.userId}')
@ -2874,7 +2878,7 @@ App::post('/v1/account/verification/phone')
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
->label('sdk.response.model', Response::MODEL_TOKEN)
->label('abuse-limit', 10)
->label('abuse-key', 'userId:{userId}')
->label('abuse-key', ['url:{url},userId:{userId}', 'url:{url},ip:{ip}'])
->inject('request')
->inject('response')
->inject('user')
@ -2935,9 +2939,12 @@ App::post('/v1/account/verification/phone')
$message = $message->setParam('{{token}}', $secret);
$message = $message->render();
var_dump($request->getIP());
var_dump($project->getId());
$queueForMessaging
->setRecipient($user->getAttribute('phone'))
->setMessage($message)
->setProject($project)
->trigger()
;

115
app/controllers/api/health.php
View file

@ -7,6 +7,7 @@ use Appwrite\Utopia\Response;
use Utopia\App;
use Utopia\Config\Config;
use Utopia\Database\Document;
use Utopia\Domains\Validator\PublicDomain;
use Utopia\Pools\Group;
use Utopia\Queue\Client;
use Utopia\Queue\Connection;
@ -14,8 +15,11 @@ use Utopia\Registry\Registry;
use Utopia\Storage\Device;
use Utopia\Storage\Device\Local;
use Utopia\Storage\Storage;
use Utopia\Validator\Domain;
use Utopia\Validator\Integer;
use Utopia\Validator\Multiple;
use Utopia\Validator\Text;
use Utopia\Validator\WhiteList;
App::get('/v1/health')
->desc('Get HTTP')
@ -355,7 +359,7 @@ App::get('/v1/health/queue/webhooks')
$size = $client->getQueueSize();
if ($size >= $threshold) {
throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
throw new Exception(Exception::HEALTH_QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
}
$response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE);
@ -382,12 +386,62 @@ App::get('/v1/health/queue/logs')
$size = $client->getQueueSize();
if ($size >= $threshold) {
throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
throw new Exception(Exception::HEALTH_QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
}
$response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE);
}, ['response']);
App::get('/v1/health/certificate')
->desc('Get the SSL certificate for a domain')
->groups(['api', 'health'])
->label('scope', 'health.read')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'health')
->label('sdk.method', 'getCertificate')
->label('sdk.description', '/docs/references/health/get-certificate.md')
->label('sdk.response.code', Response::STATUS_CODE_OK)
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
->label('sdk.response.model', Response::MODEL_HEALTH_CERTIFICATE)
->param('domain', null, new Multiple([new Domain(), new PublicDomain()]), Multiple::TYPE_STRING, 'Domain name')
->inject('response')
->action(function (string $domain, Response $response) {
if (filter_var($domain, FILTER_VALIDATE_URL)) {
$domain = parse_url($domain, PHP_URL_HOST);
}
$sslContext = stream_context_create([
"ssl" => [
"capture_peer_cert" => true
]
]);
$sslSocket = stream_socket_client("ssl://" . $domain . ":443", $errno, $errstr, 30, STREAM_CLIENT_CONNECT, $sslContext);
if (!$sslSocket) {
throw new Exception(Exception::HEALTH_INVALID_HOST);
}
$streamContextParams = stream_context_get_params($sslSocket);
$peerCertificate = $streamContextParams['options']['ssl']['peer_certificate'];
$certificatePayload = openssl_x509_parse($peerCertificate);
$sslExpiration = $certificatePayload['validTo_time_t'];
$status = ($sslExpiration < time()) ? 'fail' : 'pass';
if ($status == 'fail') {
throw new Exception(Exception::HEALTH_CERTIFICATE_EXPIRED);
}
$response->dynamic(new Document([
'name' => $certificatePayload['name'],
'subjectSN' => $certificatePayload['subject']['CN'],
'issuerOrganisation' => $certificatePayload['issuer']['O'],
'validFrom' => $certificatePayload['validFrom_time_t'],
'validTo' => $certificatePayload['validTo_time_t'],
'signatureTypeSN' => $certificatePayload['signatureTypeSN'],
]), Response::MODEL_HEALTH_CERTIFICATE);
}, ['response']);
App::get('/v1/health/queue/certificates')
->desc('Get certificates queue')
->groups(['api', 'health'])
@ -409,7 +463,7 @@ App::get('/v1/health/queue/certificates')
$size = $client->getQueueSize();
if ($size >= $threshold) {
throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
throw new Exception(Exception::HEALTH_QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
}
$response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE);
@ -436,7 +490,7 @@ App::get('/v1/health/queue/builds')
$size = $client->getQueueSize();
if ($size >= $threshold) {
throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
throw new Exception(Exception::HEALTH_QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
}
$response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE);
@ -464,7 +518,7 @@ App::get('/v1/health/queue/databases')
$size = $client->getQueueSize();
if ($size >= $threshold) {
throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
throw new Exception(Exception::HEALTH_QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
}
$response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE);
@ -491,7 +545,7 @@ App::get('/v1/health/queue/deletes')
$size = $client->getQueueSize();
if ($size >= $threshold) {
throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
throw new Exception(Exception::HEALTH_QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
}
$response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE);
@ -518,7 +572,7 @@ App::get('/v1/health/queue/mails')
$size = $client->getQueueSize();
if ($size >= $threshold) {
throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
throw new Exception(Exception::HEALTH_QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
}
$response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE);
@ -545,7 +599,7 @@ App::get('/v1/health/queue/messaging')
$size = $client->getQueueSize();
if ($size >= $threshold) {
throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
throw new Exception(Exception::HEALTH_QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
}
$response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE);
@ -572,7 +626,7 @@ App::get('/v1/health/queue/migrations')
$size = $client->getQueueSize();
if ($size >= $threshold) {
throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
throw new Exception(Exception::HEALTH_QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
}
$response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE);
@ -599,7 +653,7 @@ App::get('/v1/health/queue/functions')
$size = $client->getQueueSize();
if ($size >= $threshold) {
throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
throw new Exception(Exception::HEALTH_QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
}
$response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE);
@ -687,6 +741,47 @@ App::get('/v1/health/anti-virus')
$response->dynamic(new Document($output), Response::MODEL_HEALTH_ANTIVIRUS);
});
App::get('/v1/health/queue/failed/:name')
->desc('Get number of failed queue jobs')
->groups(['api', 'health'])
->label('scope', 'health.read')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'health')
->label('sdk.method', 'getFailedJobs')
->param('name', '', new WhiteList([
Event::DATABASE_QUEUE_NAME,
Event::DELETE_QUEUE_NAME,
Event::AUDITS_QUEUE_NAME,
Event::MAILS_QUEUE_NAME,
Event::FUNCTIONS_QUEUE_NAME,
Event::USAGE_QUEUE_NAME,
Event::WEBHOOK_CLASS_NAME,
Event::CERTIFICATES_QUEUE_NAME,
Event::BUILDS_QUEUE_NAME,
Event::MESSAGING_QUEUE_NAME,
Event::MIGRATIONS_QUEUE_NAME,
Event::HAMSTER_CLASS_NAME
]), 'The name of the queue')
->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true)
->label('sdk.description', '/docs/references/health/get-failed-queue-jobs.md')
->label('sdk.response.code', Response::STATUS_CODE_OK)
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE)
->inject('response')
->inject('queue')
->action(function (string $name, int|string $threshold, Response $response, Connection $queue) {
$threshold = \intval($threshold);
$client = new Client($name, $queue);
$failed = $client->countFailedJobs();
if ($failed >= $threshold) {
throw new Exception(Exception::HEALTH_QUEUE_SIZE_EXCEEDED, "Queue failed jobs threshold hit. Current size is {$failed} and threshold is {$threshold}.");
}
$response->dynamic(new Document([ 'size' => $failed ]), Response::MODEL_HEALTH_QUEUE);
});
App::get('/v1/health/stats') // Currently only used internally
->desc('Get system stats')
->groups(['api', 'health'])

11
app/controllers/api/proxy.php
View file

@ -14,6 +14,7 @@ use Utopia\Database\Helpers\ID;
use Utopia\Database\Query;
use Utopia\Database\Validator\UID;
use Utopia\Domains\Domain;
use Utopia\Logger\Log;
use Utopia\Validator\Domain as ValidatorDomain;
use Utopia\Validator\Text;
use Utopia\Validator\WhiteList;
@ -278,7 +279,8 @@ App::patch('/v1/proxy/rules/:ruleId/verification')
->inject('queueForEvents')
->inject('project')
->inject('dbForConsole')
->action(function (string $ruleId, Response $response, Certificate $queueForCertificates, Event $queueForEvents, Document $project, Database $dbForConsole) {
->inject('log')
->action(function (string $ruleId, Response $response, Certificate $queueForCertificates, Event $queueForEvents, Document $project, Database $dbForConsole, Log $log) {
$rule = $dbForConsole->getDocument('rules', $ruleId);
if ($rule->isEmpty() || $rule->getAttribute('projectInternalId') !== $project->getInternalId()) {
@ -298,7 +300,14 @@ App::patch('/v1/proxy/rules/:ruleId/verification')
$validator = new CNAME($target->get()); // Verify Domain with DNS records
$domain = new Domain($rule->getAttribute('domain', ''));
$validationStart = \microtime(true);
if (!$validator->isValid($domain->get())) {
$log->addExtra('dnsTiming', \strval(\microtime(true) - $validationStart));
$log->addTag('dnsDomain', $domain->get());
$error = $validator->getLogs();
$log->addExtra('dnsResponse', \is_array($error) ? \json_encode($error) : \strval($error));
throw new Exception(Exception::RULE_VERIFICATION_FAILED);
}

10
app/controllers/api/teams.php
View file

@ -380,6 +380,7 @@ App::post('/v1/teams/:teamId/memberships')
->param('roles', [], new ArrayList(new Key(), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Array of strings. Use this param to set the user roles in the team. A role can be any string. Learn more about [roles and permissions](https://appwrite.io/docs/permissions). Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' roles are allowed, each 32 characters long.')
->param('url', '', fn($clients) => new Host($clients), 'URL to redirect the user back to your app from the invitation email. Only URLs from hostnames in your project platform list are allowed. This requirement helps to prevent an [open redirect](https://cheatsheetseries.owasp.org/cheatsheets/Unvalidated_Redirects_and_Forwards_Cheat_Sheet.html) attack against your project API.', true, ['clients']) // TODO add our own built-in confirm page
->param('name', '', new Text(128), 'Name of the new team member. Max length: 128 chars.', true)
->inject('request')
->inject('response')
->inject('project')
->inject('user')
@ -388,7 +389,7 @@ App::post('/v1/teams/:teamId/memberships')
->inject('queueForMails')
->inject('queueForMessaging')
->inject('queueForEvents')
->action(function (string $teamId, string $email, string $userId, string $phone, array $roles, string $url, string $name, Response $response, Document $project, Document $user, Database $dbForProject, Locale $locale, Mail $queueForMails, EventPhone $queueForMessaging, Event $queueForEvents) {
->action(function (string $teamId, string $email, string $userId, string $phone, array $roles, string $url, string $name, Request $request, Response $response, Document $project, Document $user, Database $dbForProject, Locale $locale, Mail $queueForMails, EventPhone $queueForMessaging, Event $queueForEvents) {
$isAPIKey = Auth::isAppUser(Authorization::getRoles());
$isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles());
@ -533,8 +534,8 @@ App::post('/v1/teams/:teamId/memberships')
} catch (Duplicate $th) {
throw new Exception(Exception::TEAM_INVITE_ALREADY_EXISTS);
}
$team->setAttribute('total', $team->getAttribute('total', 0) + 1);
$team = Authorization::skip(fn() => $dbForProject->updateDocument('teams', $team->getId(), $team));
Authorization::skip(fn() => $dbForProject->increaseDocumentAttribute('teams', $team->getId(), 'total', 1));
$dbForProject->deleteCachedDocument('users', $invitee->getId());
} else {
@ -638,9 +639,12 @@ App::post('/v1/teams/:teamId/memberships')
$message = $message->setParam('{{token}}', $url);
$message = $message->render();
var_dump($request->getIP());
var_dump($project->getId());
$queueForMessaging
->setRecipient($phone)
->setMessage($message)
->setProject($project)
->trigger();
}
}

96
app/controllers/general.php
View file

@ -603,66 +603,58 @@ App::error()
->inject('response')
->inject('project')
->inject('logger')
->inject('loggerBreadcrumbs')
->action(function (Throwable $error, App $utopia, Request $request, Response $response, Document $project, ?Logger $logger, array $loggerBreadcrumbs) {
->inject('log')
->action(function (Throwable $error, App $utopia, Request $request, Response $response, Document $project, ?Logger $logger, Log $log) {
$version = App::getEnv('_APP_VERSION', 'UNKNOWN');
$route = $utopia->getRoute();
$publish = true;
if ($error instanceof AppwriteException) {
$publish = $error->isPublishable();
} else {
$publish = $error->getCode() === 0 || $error->getCode() >= 500;
}
if ($logger && $publish) {
if ($error->getCode() >= 500 || $error->getCode() === 0) {
try {
/** @var Utopia\Database\Document $user */
$user = $utopia->getResource('user');
} catch (\Throwable $th) {
// All good, user is optional information for logger
}
$log = new Utopia\Logger\Log();
if (isset($user) && !$user->isEmpty()) {
$log->setUser(new User($user->getId()));
}
$log->setNamespace("http");
$log->setServer(\gethostname());
$log->setVersion($version);
$log->setType(Log::TYPE_ERROR);
$log->setMessage($error->getMessage());
$log->addTag('database', $project->getAttribute('database', 'console'));
$log->addTag('method', $route->getMethod());
$log->addTag('url', $route->getPath());
$log->addTag('verboseType', get_class($error));
$log->addTag('code', $error->getCode());
$log->addTag('projectId', $project->getId());
$log->addTag('hostname', $request->getHostname());
$log->addTag('locale', (string)$request->getParam('locale', $request->getHeader('x-appwrite-locale', '')));
$log->addExtra('file', $error->getFile());
$log->addExtra('line', $error->getLine());
$log->addExtra('trace', $error->getTraceAsString());
$log->addExtra('detailedTrace', $error->getTrace());
$log->addExtra('roles', Authorization::getRoles());
$action = $route->getLabel("sdk.namespace", "UNKNOWN_NAMESPACE") . '.' . $route->getLabel("sdk.method", "UNKNOWN_METHOD");
$log->setAction($action);
$isProduction = App::getEnv('_APP_ENV', 'development') === 'production';
$log->setEnvironment($isProduction ? Log::ENVIRONMENT_PRODUCTION : Log::ENVIRONMENT_STAGING);
foreach ($loggerBreadcrumbs as $loggerBreadcrumb) {
$log->addBreadcrumb($loggerBreadcrumb);
}
$responseCode = $logger->addLog($log);
Console::info('Log pushed with status code: ' . $responseCode);
if ($logger && ($publish || $error->getCode() === 0)) {
try {
/** @var Utopia\Database\Document $user */
$user = $utopia->getResource('user');
} catch (\Throwable $th) {
// All good, user is optional information for logger
}
if (isset($user) && !$user->isEmpty()) {
$log->setUser(new User($user->getId()));
}
$log->setNamespace("http");
$log->setServer(\gethostname());
$log->setVersion($version);
$log->setType(Log::TYPE_ERROR);
$log->setMessage($error->getMessage());
$log->addTag('database', $project->getAttribute('database', 'console'));
$log->addTag('method', $route->getMethod());
$log->addTag('url', $route->getPath());
$log->addTag('verboseType', get_class($error));
$log->addTag('code', $error->getCode());
$log->addTag('projectId', $project->getId());
$log->addTag('hostname', $request->getHostname());
$log->addTag('locale', (string)$request->getParam('locale', $request->getHeader('x-appwrite-locale', '')));
$log->addExtra('file', $error->getFile());
$log->addExtra('line', $error->getLine());
$log->addExtra('trace', $error->getTraceAsString());
$log->addExtra('detailedTrace', $error->getTrace());
$log->addExtra('roles', Authorization::getRoles());
$action = $route->getLabel("sdk.namespace", "UNKNOWN_NAMESPACE") . '.' . $route->getLabel("sdk.method", "UNKNOWN_METHOD");
$log->setAction($action);
$isProduction = App::getEnv('_APP_ENV', 'development') === 'production';
$log->setEnvironment($isProduction ? Log::ENVIRONMENT_PRODUCTION : Log::ENVIRONMENT_STAGING);
$responseCode = $logger->addLog($log);
Console::info('Log pushed with status code: ' . $responseCode);
}
$code = $error->getCode();

11
app/controllers/shared/api.php
View file

@ -177,6 +177,7 @@ App::init()
$end = $request->getContentRangeEnd();
$timeLimit = new TimeLimit($abuseKey, $route->getLabel('abuse-limit', 0), $route->getLabel('abuse-time', 3600), $dbForProject);
$timeLimit
->setParam('{projectId}', $project->getId())
->setParam('{userId}', $user->getId())
->setParam('{userAgent}', $request->getUserAgent(''))
->setParam('{ip}', $request->getIP())
@ -335,7 +336,7 @@ App::init()
break;
case 'magic-url':
if ($project->getAttribute('usersAuthMagicURL', true) === false) {
if (($auths['usersAuthMagicURL'] ?? true) === false) {
throw new Exception(Exception::USER_AUTH_METHOD_UNSUPPORTED, 'Magic URL authentication is disabled for this project');
}
break;
@ -346,6 +347,12 @@ App::init()
}
break;
case 'phone':
if (($auths['phone'] ?? true) === false) {
throw new Exception(Exception::USER_AUTH_METHOD_UNSUPPORTED, 'Phone authentication is disabled for this project');
}
break;
case 'invites':
if (($auths['invites'] ?? true) === false) {
throw new Exception(Exception::USER_AUTH_METHOD_UNSUPPORTED, 'Invites authentication is disabled for this project');
@ -359,7 +366,7 @@ App::init()
break;
default:
throw new Exception(Exception::USER_AUTH_METHOD_UNSUPPORTED, 'Unsupported authentication route');
throw new Exception(Exception::USER_AUTH_METHOD_UNSUPPORTED, 'Unsupported authentication type: ' . $route->getLabel('auth.type', ''));
break;
}
});

8
app/controllers/shared/api/auth.php
View file

@ -32,7 +32,7 @@ App::init()
break;
case 'magic-url':
if ($project->getAttribute('usersAuthMagicURL', true) === false) {
if (($auths['usersAuthMagicURL'] ?? true) === false) {
throw new Exception(Exception::USER_AUTH_METHOD_UNSUPPORTED, 'Magic URL authentication is disabled for this project');
}
break;
@ -43,6 +43,12 @@ App::init()
}
break;
case 'phone':
if (($auths['phone'] ?? true) === false) {
throw new Exception(Exception::USER_AUTH_METHOD_UNSUPPORTED, 'Phone authentication is disabled for this project');
}
break;
case 'invites':
if (($auths['invites'] ?? true) === false) {
throw new Exception(Exception::USER_AUTH_METHOD_UNSUPPORTED, 'Invites authentication is disabled for this project');

7
app/http.php
View file

@ -263,10 +263,9 @@ $http->on('request', function (SwooleRequest $swooleRequest, SwooleResponse $swo
// All good, user is optional information for logger
}
$loggerBreadcrumbs = $app->getResource("loggerBreadcrumbs");
$route = $app->getRoute();
$log = new Utopia\Logger\Log();
$log = $app->getResource("log");
if (isset($user) && !$user->isEmpty()) {
$log->setUser(new User($user->getId()));
@ -298,10 +297,6 @@ $http->on('request', function (SwooleRequest $swooleRequest, SwooleResponse $swo
$isProduction = App::getEnv('_APP_ENV', 'development') === 'production';
$log->setEnvironment($isProduction ? Log::ENVIRONMENT_PRODUCTION : Log::ENVIRONMENT_STAGING);
foreach ($loggerBreadcrumbs as $loggerBreadcrumb) {
$log->addBreadcrumb($loggerBreadcrumb);
}
$responseCode = $logger->addLog($log);
Console::info('Log pushed with status code: ' . $responseCode);
}

7
app/init.php
View file

@ -76,6 +76,7 @@ use Appwrite\Hooks\Hooks;
use MaxMind\Db\Reader;
use PHPMailer\PHPMailer\PHPMailer;
use Swoole\Database\PDOProxy;
use Utopia\Logger\Log;
use Utopia\Queue;
use Utopia\Queue\Connection;
use Utopia\Storage\Storage;
@ -194,6 +195,7 @@ const FUNCTION_ALLOWLIST_HEADERS_RESPONSE = ['content-type', 'content-length'];
// Usage metrics
const METRIC_TEAMS = 'teams';
const METRIC_USERS = 'users';
const METRIC_MESSAGES = 'messages';
const METRIC_SESSIONS = 'sessions';
const METRIC_DATABASES = 'databases';
const METRIC_COLLECTIONS = 'collections';
@ -864,6 +866,7 @@ foreach ($locales as $locale) {
]);
// Runtime Execution
App::setResource('log', fn() => new Log());
App::setResource('logger', function ($register) {
return $register->get('logger');
}, ['register']);
@ -872,10 +875,6 @@ App::setResource('hooks', function ($register) {
return $register->get('hooks');
}, ['register']);
App::setResource('loggerBreadcrumbs', function () {
return [];
});
App::setResource('register', fn() => $register);
App::setResource('locale', fn() => new Locale(App::getEnv('_APP_LOCALE', 'en')));

1
app/views/install/compose.phtml
View file

@ -143,6 +143,7 @@ services:
- _APP_LOGGING_PROVIDER
- _APP_LOGGING_CONFIG
- _APP_MAINTENANCE_INTERVAL
- _APP_MAINTENANCE_DELAY
- _APP_MAINTENANCE_RETENTION_EXECUTION
- _APP_MAINTENANCE_RETENTION_CACHE
- _APP_MAINTENANCE_RETENTION_ABUSE

3
bin/queue-count-failed Normal file
View file

@ -0,0 +1,3 @@
#!/bin/sh
php /usr/src/code/app/cli.php queue-count --type=failed $@

3
bin/queue-count-processing Normal file
View file

@ -0,0 +1,3 @@
#!/bin/sh
php /usr/src/code/app/cli.php queue-count --type=processing $@

3
bin/queue-count-success Normal file
View file

@ -0,0 +1,3 @@
#!/bin/sh
php /usr/src/code/app/cli.php queue-count --type=success $@

3
bin/queue-retry Normal file
View file

@ -0,0 +1,3 @@
#!/bin/sh
php /usr/src/code/app/cli.php queue-retry $@

20
composer.json
View file

@ -45,24 +45,24 @@
"appwrite/php-clamav": "2.0.*",
"utopia-php/abuse": "0.33.*",
"utopia-php/analytics": "0.10.*",
"utopia-php/audit": "0.38.*",
"utopia-php/audit": "0.35.*",
"utopia-php/cache": "0.9.*",
"utopia-php/cli": "0.15.*",
"utopia-php/config": "0.2.*",
"utopia-php/database": "0.45.6",
"utopia-php/database": "0.45.*",
"utopia-php/domains": "0.5.*",
"utopia-php/dsn": "0.2.*",
"utopia-php/dsn": "0.1.*",
"utopia-php/framework": "0.33.*",
"utopia-php/image": "0.6.*",
"utopia-php/image": "0.5.*",
"utopia-php/locale": "0.4.*",
"utopia-php/logger": "0.3.*",
"utopia-php/messaging": "0.9.*",
"utopia-php/messaging": "0.2.*",
"utopia-php/migration": "0.3.*",
"utopia-php/orchestration": "0.9.*",
"utopia-php/platform": "0.5.*",
"utopia-php/pools": "0.4.*",
"utopia-php/preloader": "0.2.*",
"utopia-php/queue": "0.6.*",
"utopia-php/queue": "0.7.*",
"utopia-php/registry": "0.5.*",
"utopia-php/storage": "0.18.*",
"utopia-php/swoole": "0.5.*",
@ -73,9 +73,8 @@
"phpmailer/phpmailer": "6.8.0",
"chillerlan/php-qrcode": "4.3.4",
"adhocore/jwt": "1.1.2",
"spomky-labs/otphp": "^10.0",
"webonyx/graphql-php": "14.11.*",
"league/csv": "^9.14"
"league/csv": "9.7.1"
},
"repositories": [
{
@ -89,15 +88,14 @@
"phpunit/phpunit": "9.5.20",
"squizlabs/php_codesniffer": "^3.7",
"swoole/ide-helper": "5.0.2",
"textalk/websocket": "1.5.7",
"utopia-php/fetch": "0.1.*"
"textalk/websocket": "1.5.7"
},
"provide": {
"ext-phpiredis": "*"
},
"config": {
"platform": {
"php": "8.2"
"php": "8.0"
}
}
}

533
composer.lock generated
View file

@ -4,7 +4,7 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
"This file is @generated automatically"
],
"content-hash": "b70152e4a3c53762b42d9623ea204cf6",
"content-hash": "fd03f97115d752d1a94b533ccf570109",
"packages": [
{
"name": "adhocore/jwt",
@ -197,73 +197,6 @@
],
"time": "2023-11-22T15:36:00+00:00"
},
{
"name": "beberlei/assert",
"version": "v3.3.2",
"source": {
"type": "git",
"url": "https://github.com/beberlei/assert.git",
"reference": "cb70015c04be1baee6f5f5c953703347c0ac1655"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/beberlei/assert/zipball/cb70015c04be1baee6f5f5c953703347c0ac1655",
"reference": "cb70015c04be1baee6f5f5c953703347c0ac1655",
"shasum": ""
},
"require": {
"ext-ctype": "*",
"ext-json": "*",
"ext-mbstring": "*",
"ext-simplexml": "*",
"php": "^7.0 || ^8.0"
},
"require-dev": {
"friendsofphp/php-cs-fixer": "*",
"phpstan/phpstan": "*",
"phpunit/phpunit": ">=6.0.0",
"yoast/phpunit-polyfills": "^0.1.0"
},
"suggest": {
"ext-intl": "Needed to allow Assertion::count(), Assertion::isCountable(), Assertion::minCount(), and Assertion::maxCount() to operate on ResourceBundles"
},
"type": "library",
"autoload": {
"files": [
"lib/Assert/functions.php"
],
"psr-4": {
"Assert\\": "lib/Assert"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"BSD-2-Clause"
],
"authors": [
{
"name": "Benjamin Eberlei",
"email": "kontakt@beberlei.de",
"role": "Lead Developer"
},
{
"name": "Richard Quadling",
"email": "rquadling@gmail.com",
"role": "Collaborator"
}
],
"description": "Thin assertion library for input validation in business models.",
"keywords": [
"assert",
"assertion",
"validation"
],
"support": {
"issues": "https://github.com/beberlei/assert/issues",
"source": "https://github.com/beberlei/assert/tree/v3.3.2"
},
"time": "2021-12-16T21:41:27+00:00"
},
{
"name": "chillerlan/php-qrcode",
"version": "4.3.4",
@ -530,39 +463,34 @@
},
{
"name": "league/csv",
"version": "9.14.0",
"version": "9.7.1",
"source": {
"type": "git",
"url": "https://github.com/thephpleague/csv.git",
"reference": "34bf0df7340b60824b9449b5c526fcc3325070d5"
"reference": "0ec57e8264ec92565974ead0d1724cf1026e10c1"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/thephpleague/csv/zipball/34bf0df7340b60824b9449b5c526fcc3325070d5",
"reference": "34bf0df7340b60824b9449b5c526fcc3325070d5",
"url": "https://api.github.com/repos/thephpleague/csv/zipball/0ec57e8264ec92565974ead0d1724cf1026e10c1",
"reference": "0ec57e8264ec92565974ead0d1724cf1026e10c1",
"shasum": ""
},
"require": {
"ext-filter": "*",
"ext-json": "*",
"ext-mbstring": "*",
"php": "^8.1.2"
"php": "^7.3 || ^8.0"
},
"require-dev": {
"doctrine/collections": "^2.1.4",
"ext-curl": "*",
"ext-dom": "*",
"ext-xdebug": "*",
"friendsofphp/php-cs-fixer": "^v3.22.0",
"phpbench/phpbench": "^1.2.15",
"phpstan/phpstan": "^1.10.50",
"phpstan/phpstan-deprecation-rules": "^1.1.4",
"phpstan/phpstan-phpunit": "^1.3.15",
"phpstan/phpstan-strict-rules": "^1.5.2",
"phpunit/phpunit": "^10.5.3",
"symfony/var-dumper": "^6.4.0"
"friendsofphp/php-cs-fixer": "^2.16",
"phpstan/phpstan": "^0.12.0",
"phpstan/phpstan-phpunit": "^0.12.0",
"phpstan/phpstan-strict-rules": "^0.12.0",
"phpunit/phpunit": "^9.5"
},
"suggest": {
"ext-dom": "Required to use the XMLConverter and the HTMLConverter classes",
"ext-dom": "Required to use the XMLConverter and or the HTMLConverter classes",
"ext-iconv": "Needed to ease transcoding CSV using iconv stream filters"
},
"type": "library",
@ -592,7 +520,7 @@
}
],
"description": "CSV data manipulation made easy in PHP",
"homepage": "https://csv.thephpleague.com",
"homepage": "http://csv.thephpleague.com",
"keywords": [
"convert",
"csv",
@ -615,7 +543,7 @@
"type": "github"
}
],
"time": "2023-12-29T07:34:53+00:00"
"time": "2021-04-17T16:32:08+00:00"
},
{
"name": "matomo/device-detector",
@ -805,73 +733,6 @@
],
"time": "2019-09-10T13:16:29+00:00"
},
{
"name": "paragonie/constant_time_encoding",
"version": "v2.6.3",
"source": {
"type": "git",
"url": "https://github.com/paragonie/constant_time_encoding.git",
"reference": "58c3f47f650c94ec05a151692652a868995d2938"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/paragonie/constant_time_encoding/zipball/58c3f47f650c94ec05a151692652a868995d2938",
"reference": "58c3f47f650c94ec05a151692652a868995d2938",
"shasum": ""
},
"require": {
"php": "^7|^8"
},
"require-dev": {
"phpunit/phpunit": "^6|^7|^8|^9",
"vimeo/psalm": "^1|^2|^3|^4"
},
"type": "library",
"autoload": {
"psr-4": {
"ParagonIE\\ConstantTime\\": "src/"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Paragon Initiative Enterprises",
"email": "security@paragonie.com",
"homepage": "https://paragonie.com",
"role": "Maintainer"
},
{
"name": "Steve 'Sc00bz' Thomas",
"email": "steve@tobtu.com",
"homepage": "https://www.tobtu.com",
"role": "Original Developer"
}
],
"description": "Constant-time Implementations of RFC 4648 Encoding (Base-64, Base-32, Base-16)",
"keywords": [
"base16",
"base32",
"base32_decode",
"base32_encode",
"base64",
"base64_decode",
"base64_encode",
"bin2hex",
"encoding",
"hex",
"hex2bin",
"rfc4648"
],
"support": {
"email": "info@paragonie.com",
"issues": "https://github.com/paragonie/constant_time_encoding/issues",
"source": "https://github.com/paragonie/constant_time_encoding"
},
"time": "2022-06-14T06:56:20+00:00"
},
{
"name": "phpmailer/phpmailer",
"version": "v6.8.0",
@ -952,93 +813,18 @@
],
"time": "2023-03-06T14:43:22+00:00"
},
{
"name": "spomky-labs/otphp",
"version": "v10.0.3",
"source": {
"type": "git",
"url": "https://github.com/Spomky-Labs/otphp.git",
"reference": "9784d9f7c790eed26e102d6c78f12c754036c366"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/Spomky-Labs/otphp/zipball/9784d9f7c790eed26e102d6c78f12c754036c366",
"reference": "9784d9f7c790eed26e102d6c78f12c754036c366",
"shasum": ""
},
"require": {
"beberlei/assert": "^3.0",
"ext-mbstring": "*",
"paragonie/constant_time_encoding": "^2.0",
"php": "^7.2|^8.0",
"thecodingmachine/safe": "^0.1.14|^1.0|^2.0"
},
"require-dev": {
"php-coveralls/php-coveralls": "^2.0",
"phpstan/phpstan": "^0.12",
"phpstan/phpstan-beberlei-assert": "^0.12",
"phpstan/phpstan-deprecation-rules": "^0.12",
"phpstan/phpstan-phpunit": "^0.12",
"phpstan/phpstan-strict-rules": "^0.12",
"phpunit/phpunit": "^8.0",
"thecodingmachine/phpstan-safe-rule": "^1.0 || ^2.0"
},
"type": "library",
"extra": {
"branch-alias": {
"v10.0": "10.0.x-dev",
"v9.0": "9.0.x-dev",
"v8.3": "8.3.x-dev"
}
},
"autoload": {
"psr-4": {
"OTPHP\\": "src/"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Florent Morselli",
"homepage": "https://github.com/Spomky"
},
{
"name": "All contributors",
"homepage": "https://github.com/Spomky-Labs/otphp/contributors"
}
],
"description": "A PHP library for generating one time passwords according to RFC 4226 (HOTP Algorithm) and the RFC 6238 (TOTP Algorithm) and compatible with Google Authenticator",
"homepage": "https://github.com/Spomky-Labs/otphp",
"keywords": [
"FreeOTP",
"RFC 4226",
"RFC 6238",
"google authenticator",
"hotp",
"otp",
"totp"
],
"support": {
"issues": "https://github.com/Spomky-Labs/otphp/issues",
"source": "https://github.com/Spomky-Labs/otphp/tree/v10.0.3"
},
"time": "2022-03-17T08:00:35+00:00"
},
{
"name": "symfony/polyfill-php80",
"version": "v1.28.0",
"version": "v1.29.0",
"source": {
"type": "git",
"url": "https://github.com/symfony/polyfill-php80.git",
"reference": "6caa57379c4aec19c0a12a38b59b26487dcfe4b5"
"reference": "87b68208d5c1188808dd7839ee1e6c8ec3b02f1b"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/symfony/polyfill-php80/zipball/6caa57379c4aec19c0a12a38b59b26487dcfe4b5",
"reference": "6caa57379c4aec19c0a12a38b59b26487dcfe4b5",
"url": "https://api.github.com/repos/symfony/polyfill-php80/zipball/87b68208d5c1188808dd7839ee1e6c8ec3b02f1b",
"reference": "87b68208d5c1188808dd7839ee1e6c8ec3b02f1b",
"shasum": ""
},
"require": {
@ -1046,9 +832,6 @@
},
"type": "library",
"extra": {
"branch-alias": {
"dev-main": "1.28-dev"
},
"thanks": {
"name": "symfony/polyfill",
"url": "https://github.com/symfony/polyfill"
@ -1092,7 +875,7 @@
"shim"
],
"support": {
"source": "https://github.com/symfony/polyfill-php80/tree/v1.28.0"
"source": "https://github.com/symfony/polyfill-php80/tree/v1.29.0"
},
"funding": [
{
@ -1108,146 +891,7 @@
"type": "tidelift"
}
],
"time": "2023-01-26T09:26:14+00:00"
},
{
"name": "thecodingmachine/safe",
"version": "v2.5.0",
"source": {
"type": "git",
"url": "https://github.com/thecodingmachine/safe.git",
"reference": "3115ecd6b4391662b4931daac4eba6b07a2ac1f0"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/thecodingmachine/safe/zipball/3115ecd6b4391662b4931daac4eba6b07a2ac1f0",
"reference": "3115ecd6b4391662b4931daac4eba6b07a2ac1f0",
"shasum": ""
},
"require": {
"php": "^8.0"
},
"require-dev": {
"phpstan/phpstan": "^1.5",
"phpunit/phpunit": "^9.5",
"squizlabs/php_codesniffer": "^3.2",
"thecodingmachine/phpstan-strict-rules": "^1.0"
},
"type": "library",
"extra": {
"branch-alias": {
"dev-master": "2.2.x-dev"
}
},
"autoload": {
"files": [
"deprecated/apc.php",
"deprecated/array.php",
"deprecated/datetime.php",
"deprecated/libevent.php",
"deprecated/misc.php",
"deprecated/password.php",
"deprecated/mssql.php",
"deprecated/stats.php",
"deprecated/strings.php",
"lib/special_cases.php",
"deprecated/mysqli.php",
"generated/apache.php",
"generated/apcu.php",
"generated/array.php",
"generated/bzip2.php",
"generated/calendar.php",
"generated/classobj.php",
"generated/com.php",
"generated/cubrid.php",
"generated/curl.php",
"generated/datetime.php",
"generated/dir.php",
"generated/eio.php",
"generated/errorfunc.php",
"generated/exec.php",
"generated/fileinfo.php",
"generated/filesystem.php",
"generated/filter.php",
"generated/fpm.php",
"generated/ftp.php",
"generated/funchand.php",
"generated/gettext.php",
"generated/gmp.php",
"generated/gnupg.php",
"generated/hash.php",
"generated/ibase.php",
"generated/ibmDb2.php",
"generated/iconv.php",
"generated/image.php",
"generated/imap.php",
"generated/info.php",
"generated/inotify.php",
"generated/json.php",
"generated/ldap.php",
"generated/libxml.php",
"generated/lzf.php",
"generated/mailparse.php",
"generated/mbstring.php",
"generated/misc.php",
"generated/mysql.php",
"generated/network.php",
"generated/oci8.php",
"generated/opcache.php",
"generated/openssl.php",
"generated/outcontrol.php",
"generated/pcntl.php",
"generated/pcre.php",
"generated/pgsql.php",
"generated/posix.php",
"generated/ps.php",
"generated/pspell.php",
"generated/readline.php",
"generated/rpminfo.php",
"generated/rrd.php",
"generated/sem.php",
"generated/session.php",
"generated/shmop.php",
"generated/sockets.php",
"generated/sodium.php",
"generated/solr.php",
"generated/spl.php",
"generated/sqlsrv.php",
"generated/ssdeep.php",
"generated/ssh2.php",
"generated/stream.php",
"generated/strings.php",
"generated/swoole.php",
"generated/uodbc.php",
"generated/uopz.php",
"generated/url.php",
"generated/var.php",
"generated/xdiff.php",
"generated/xml.php",
"generated/xmlrpc.php",
"generated/yaml.php",
"generated/yaz.php",
"generated/zip.php",
"generated/zlib.php"
],
"classmap": [
"lib/DateTime.php",
"lib/DateTimeImmutable.php",
"lib/Exceptions/",
"deprecated/Exceptions/",
"generated/Exceptions/"
]
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"description": "PHP core functions that throw exceptions instead of returning FALSE on error",
"support": {
"issues": "https://github.com/thecodingmachine/safe/issues",
"source": "https://github.com/thecodingmachine/safe/tree/v2.5.0"
},
"time": "2023-04-05T11:54:14+00:00"
"time": "2024-01-29T20:11:03+00:00"
},
{
"name": "utopia-php/abuse",
@ -1659,16 +1303,16 @@
},
{
"name": "utopia-php/dsn",
"version": "0.2.0",
"version": "0.1.0",
"source": {
"type": "git",
"url": "https://github.com/utopia-php/dsn.git",
"reference": "c11f37a12c3f6aaf9fea97ca7cb363dcc93668d7"
"reference": "17a5935eab1b89fb4b95600db50a1b6d5faa6cea"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/utopia-php/dsn/zipball/c11f37a12c3f6aaf9fea97ca7cb363dcc93668d7",
"reference": "c11f37a12c3f6aaf9fea97ca7cb363dcc93668d7",
"url": "https://api.github.com/repos/utopia-php/dsn/zipball/17a5935eab1b89fb4b95600db50a1b6d5faa6cea",
"reference": "17a5935eab1b89fb4b95600db50a1b6d5faa6cea",
"shasum": ""
},
"require": {
@ -1700,9 +1344,9 @@
],
"support": {
"issues": "https://github.com/utopia-php/dsn/issues",
"source": "https://github.com/utopia-php/dsn/tree/0.2.0"
"source": "https://github.com/utopia-php/dsn/tree/0.1.0"
},
"time": "2023-11-02T12:01:43+00:00"
"time": "2022-10-26T10:06:20+00:00"
},
{
"name": "utopia-php/framework",
@ -1751,16 +1395,16 @@
},
{
"name": "utopia-php/image",
"version": "0.6.0",
"version": "0.5.4",
"source": {
"type": "git",
"url": "https://github.com/utopia-php/image.git",
"reference": "88f7209172bdabd81e76ac981c95fac117dc6e08"
"reference": "ca5f436f9aa22dedaa6648f24f3687733808e336"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/utopia-php/image/zipball/88f7209172bdabd81e76ac981c95fac117dc6e08",
"reference": "88f7209172bdabd81e76ac981c95fac117dc6e08",
"url": "https://api.github.com/repos/utopia-php/image/zipball/ca5f436f9aa22dedaa6648f24f3687733808e336",
"reference": "ca5f436f9aa22dedaa6648f24f3687733808e336",
"shasum": ""
},
"require": {
@ -1768,8 +1412,6 @@
"php": ">=8.0"
},
"require-dev": {
"laravel/pint": "1.2.*",
"phpstan/phpstan": "1.9.x-dev",
"phpunit/phpunit": "^9.3",
"vimeo/psalm": "4.13.1"
},
@ -1783,6 +1425,12 @@
"license": [
"MIT"
],
"authors": [
{
"name": "Eldad Fux",
"email": "eldad@appwrite.io"
}
],
"description": "A simple Image manipulation library",
"keywords": [
"framework",
@ -1793,9 +1441,9 @@
],
"support": {
"issues": "https://github.com/utopia-php/image/issues",
"source": "https://github.com/utopia-php/image/tree/0.6.0"
"source": "https://github.com/utopia-php/image/tree/0.5.4"
},
"time": "2024-01-24T06:59:44+00:00"
"time": "2022-05-11T12:30:41+00:00"
},
{
"name": "utopia-php/locale",
@ -1903,28 +1551,26 @@
},
{
"name": "utopia-php/messaging",
"version": "0.9.0",
"version": "0.2.0",
"source": {
"type": "git",
"url": "https://github.com/utopia-php/messaging.git",
"reference": "df54ba51570e886724590edeb03dbd455bb0464d"
"reference": "2d0f474a106bb1da285f85e105c29b46085d3a43"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/utopia-php/messaging/zipball/df54ba51570e886724590edeb03dbd455bb0464d",
"reference": "df54ba51570e886724590edeb03dbd455bb0464d",
"url": "https://api.github.com/repos/utopia-php/messaging/zipball/2d0f474a106bb1da285f85e105c29b46085d3a43",
"reference": "2d0f474a106bb1da285f85e105c29b46085d3a43",
"shasum": ""
},
"require": {
"ext-curl": "*",
"ext-openssl": "*",
"php": ">=8.0.0"
},
"require-dev": {
"laravel/pint": "1.13.*",
"laravel/pint": "^1.2",
"phpmailer/phpmailer": "6.8.*",
"phpstan/phpstan": "1.10.*",
"phpunit/phpunit": "9.6.10"
"phpunit/phpunit": "9.6.*"
},
"type": "library",
"autoload": {
@ -1947,9 +1593,9 @@
],
"support": {
"issues": "https://github.com/utopia-php/messaging/issues",
"source": "https://github.com/utopia-php/messaging/tree/0.9.0"
"source": "https://github.com/utopia-php/messaging/tree/0.2.0"
},
"time": "2024-01-31T11:51:27+00:00"
"time": "2023-09-14T20:48:42+00:00"
},
{
"name": "utopia-php/migration",
@ -2264,16 +1910,16 @@
},
{
"name": "utopia-php/queue",
"version": "0.6.0",
"version": "0.7.0",
"source": {
"type": "git",
"url": "https://github.com/utopia-php/queue.git",
"reference": "0120bd21904cb2bee34e4571b1737589ffff0eb1"
"reference": "917565256eb94bcab7246f7a746b1a486813761b"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/utopia-php/queue/zipball/0120bd21904cb2bee34e4571b1737589ffff0eb1",
"reference": "0120bd21904cb2bee34e4571b1737589ffff0eb1",
"url": "https://api.github.com/repos/utopia-php/queue/zipball/917565256eb94bcab7246f7a746b1a486813761b",
"reference": "917565256eb94bcab7246f7a746b1a486813761b",
"shasum": ""
},
"require": {
@ -2319,9 +1965,9 @@
],
"support": {
"issues": "https://github.com/utopia-php/queue/issues",
"source": "https://github.com/utopia-php/queue/tree/0.6.0"
"source": "https://github.com/utopia-php/queue/tree/0.7.0"
},
"time": "2023-10-16T16:59:45+00:00"
"time": "2024-01-17T19:00:43+00:00"
},
{
"name": "utopia-php/registry",
@ -5123,16 +4769,16 @@
},
{
"name": "symfony/polyfill-ctype",
"version": "v1.28.0",
"version": "v1.29.0",
"source": {
"type": "git",
"url": "https://github.com/symfony/polyfill-ctype.git",
"reference": "ea208ce43cbb04af6867b4fdddb1bdbf84cc28cb"
"reference": "ef4d7e442ca910c4764bce785146269b30cb5fc4"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/symfony/polyfill-ctype/zipball/ea208ce43cbb04af6867b4fdddb1bdbf84cc28cb",
"reference": "ea208ce43cbb04af6867b4fdddb1bdbf84cc28cb",
"url": "https://api.github.com/repos/symfony/polyfill-ctype/zipball/ef4d7e442ca910c4764bce785146269b30cb5fc4",
"reference": "ef4d7e442ca910c4764bce785146269b30cb5fc4",
"shasum": ""
},
"require": {
@ -5146,9 +4792,6 @@
},
"type": "library",
"extra": {
"branch-alias": {
"dev-main": "1.28-dev"
},
"thanks": {
"name": "symfony/polyfill",
"url": "https://github.com/symfony/polyfill"
@ -5185,7 +4828,7 @@
"portable"
],
"support": {
"source": "https://github.com/symfony/polyfill-ctype/tree/v1.28.0"
"source": "https://github.com/symfony/polyfill-ctype/tree/v1.29.0"
},
"funding": [
{
@ -5201,20 +4844,20 @@
"type": "tidelift"
}
],
"time": "2023-01-26T09:26:14+00:00"
"time": "2024-01-29T20:11:03+00:00"
},
{
"name": "symfony/polyfill-mbstring",
"version": "v1.28.0",
"version": "v1.29.0",
"source": {
"type": "git",
"url": "https://github.com/symfony/polyfill-mbstring.git",
"reference": "42292d99c55abe617799667f454222c54c60e229"
"reference": "9773676c8a1bb1f8d4340a62efe641cf76eda7ec"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/symfony/polyfill-mbstring/zipball/42292d99c55abe617799667f454222c54c60e229",
"reference": "42292d99c55abe617799667f454222c54c60e229",
"url": "https://api.github.com/repos/symfony/polyfill-mbstring/zipball/9773676c8a1bb1f8d4340a62efe641cf76eda7ec",
"reference": "9773676c8a1bb1f8d4340a62efe641cf76eda7ec",
"shasum": ""
},
"require": {
@ -5228,9 +4871,6 @@
},
"type": "library",
"extra": {
"branch-alias": {
"dev-main": "1.28-dev"
},
"thanks": {
"name": "symfony/polyfill",
"url": "https://github.com/symfony/polyfill"
@ -5268,7 +4908,7 @@
"shim"
],
"support": {
"source": "https://github.com/symfony/polyfill-mbstring/tree/v1.28.0"
"source": "https://github.com/symfony/polyfill-mbstring/tree/v1.29.0"
},
"funding": [
{
@ -5284,7 +4924,7 @@
"type": "tidelift"
}
],
"time": "2023-07-28T09:04:16+00:00"
"time": "2024-01-29T20:11:03+00:00"
},
{
"name": "textalk/websocket",
@ -5456,45 +5096,6 @@
}
],
"time": "2023-11-21T18:54:41+00:00"
},
{
"name": "utopia-php/fetch",
"version": "0.1.0",
"source": {
"type": "git",
"url": "https://github.com/utopia-php/fetch.git",
"reference": "2fa214b9262acd1a3583515a364da4f35929d5c5"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/utopia-php/fetch/zipball/2fa214b9262acd1a3583515a364da4f35929d5c5",
"reference": "2fa214b9262acd1a3583515a364da4f35929d5c5",
"shasum": ""
},
"require": {
"php": ">=8.0"
},
"require-dev": {
"laravel/pint": "^1.5.0",
"phpstan/phpstan": "^1.10",
"phpunit/phpunit": "^9.5"
},
"type": "library",
"autoload": {
"psr-4": {
"Utopia\\Fetch\\": "src/"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"description": "A simple library that provides an interface for making HTTP Requests.",
"support": {
"issues": "https://github.com/utopia-php/fetch/issues",
"source": "https://github.com/utopia-php/fetch/tree/0.1.0"
},
"time": "2023-10-10T11:58:32+00:00"
}
],
"aliases": [],
@ -5521,7 +5122,7 @@
"ext-fileinfo": "*"
},
"platform-overrides": {
"php": "8.2"
"php": "8.0"
},
"plugin-api-version": "2.2.0"
"plugin-api-version": "2.3.0"
}

2
docker-compose.yml
View file

@ -572,6 +572,7 @@ services:
- _APP_REDIS_PASS
- _APP_SMS_PROVIDER
- _APP_SMS_FROM
- _APP_SMS_PROJECTS_DENY_LIST
- _APP_LOGGING_PROVIDER
- _APP_LOGGING_CONFIG
@ -645,6 +646,7 @@ services:
- _APP_MAINTENANCE_RETENTION_AUDIT
- _APP_MAINTENANCE_RETENTION_USAGE_HOURLY
- _APP_MAINTENANCE_RETENTION_SCHEDULES
- _APP_MAINTENANCE_DELAY
appwrite-worker-usage:
entrypoint: worker-usage

1
docs/references/health/get-certificate.md Normal file
View file

@ -0,0 +1 @@
Get the SSL certificate for a domain

1
docs/references/health/get-failed-queue-jobs.md Normal file
View file

@ -0,0 +1 @@
Returns the amount of failed jobs in a given queue.

17
src/Appwrite/Extend/Exception.php
View file

@ -234,25 +234,22 @@ class Exception extends \Exception
public const REALTIME_POLICY_VIOLATION = 'realtime_policy_violation';
/** Health */
public const QUEUE_SIZE_EXCEEDED = 'queue_size_exceeded';
public const HEALTH_QUEUE_SIZE_EXCEEDED = 'health_queue_size_exceeded';
public const HEALTH_CERTIFICATE_EXPIRED = 'health_certificate_expired';
public const HEALTH_INVALID_HOST = 'health_invalid_host';
protected string $type = '';
protected array $errors = [];
protected bool $publish = true;
protected bool $publish;
public function __construct(string $type = Exception::GENERAL_UNKNOWN, string $message = null, int $code = null, \Throwable $previous = null)
{
$this->errors = Config::getParam('errors');
$this->type = $type;
$this->code = $code ?? $this->errors[$type]['code'];
$this->message = $message ?? $this->errors[$type]['description'];
if (isset($this->errors[$type])) {
$this->code = $this->errors[$type]['code'];
$this->message = $this->errors[$type]['description'];
$this->publish = $this->errors[$type]['publish'] ?? true;
}
$this->message = $message ?? $this->message;
$this->code = $code ?? $this->code;
$this->publish = $this->errors[$type]['publish'] ?? ($this->code >= 500);
parent::__construct($this->message, $this->code, $previous);
}

14
src/Appwrite/Network/Validator/CNAME.php
View file

@ -6,6 +6,11 @@ use Utopia\Validator;
class CNAME extends Validator
{
/**
* @var mixed
*/
protected mixed $logs;
/**
* @var string
*/
@ -27,6 +32,14 @@ class CNAME extends Validator
return 'Invalid CNAME record';
}
/**
* @return mixed
*/
public function getLogs(): mixed
{
return $this->logs;
}
/**
* Check if CNAME record target value matches selected target
*
@ -42,6 +55,7 @@ class CNAME extends Validator
try {
$records = \dns_get_record($domain, DNS_CNAME);
$this->logs = $records;
} catch (\Throwable $th) {
return false;
}

5
src/Appwrite/Platform/Services/Tasks.php
View file

@ -20,7 +20,9 @@ use Appwrite\Platform\Tasks\Upgrade;
use Appwrite\Platform\Tasks\DeleteOrphanedProjects;
use Appwrite\Platform\Tasks\GetMigrationStats;
use Appwrite\Platform\Tasks\PatchRecreateRepositoriesDocuments;
use Appwrite\Platform\Tasks\QueueRetry;
use Appwrite\Platform\Tasks\CreateInfMetric;
use Appwrite\Platform\Tasks\QueueCount;
class Tasks extends Service
{
@ -45,8 +47,9 @@ class Tasks extends Service
->addAction(DeleteOrphanedProjects::getName(), new DeleteOrphanedProjects())
->addAction(PatchRecreateRepositoriesDocuments::getName(), new PatchRecreateRepositoriesDocuments())
->addAction(GetMigrationStats::getName(), new GetMigrationStats())
->addAction(QueueRetry::getName(), new QueueRetry())
->addAction(QueueCount::getName(), new QueueCount())
->addAction(CreateInfMetric::getName(), new CreateInfMetric())
;
}
}

3
src/Appwrite/Platform/Tasks/Maintenance.php
View file

@ -36,6 +36,7 @@ class Maintenance extends Action
// # of days in seconds (1 day = 86400s)
$interval = (int) App::getEnv('_APP_MAINTENANCE_INTERVAL', '86400');
$delay = (int) App::getEnv('_APP_MAINTENANCE_DELAY', '0');
$usageStatsRetentionHourly = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_USAGE_HOURLY', '8640000'); //100 days
$cacheRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_CACHE', '2592000'); // 30 days
$schedulesDeletionRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_SCHEDULES', '86400'); // 1 Day
@ -59,7 +60,7 @@ class Maintenance extends Action
$this->renewCertificates($dbForConsole, $queueForCertificates);
$this->notifyDeleteCache($cacheRetention, $queueForDeletes);
$this->notifyDeleteSchedules($schedulesDeletionRetention, $queueForDeletes);
}, $interval);
}, $interval, $delay);
}
protected function foreachProject(Database $dbForConsole, callable $callback): void

77
src/Appwrite/Platform/Tasks/QueueCount.php Normal file
View file

@ -0,0 +1,77 @@
<?php
namespace Appwrite\Platform\Tasks;
use Appwrite\Event\Event;
use Utopia\CLI\Console;
use Utopia\Platform\Action;
use Utopia\Queue\Client;
use Utopia\Queue\Connection;
use Utopia\Validator\WhiteList;
class QueueCount extends Action
{
public static function getName(): string
{
return 'queue-count';
}
public function __construct()
{
$this
->desc('Return the number of from a specific queue identified by the name parameter with a specific type')
->param('name', '', new WhiteList([
Event::DATABASE_QUEUE_NAME,
Event::DELETE_QUEUE_NAME,
Event::AUDITS_QUEUE_NAME,
Event::MAILS_QUEUE_NAME,
Event::FUNCTIONS_QUEUE_NAME,
Event::USAGE_QUEUE_NAME,
Event::WEBHOOK_QUEUE_NAME,
Event::CERTIFICATES_QUEUE_NAME,
Event::BUILDS_QUEUE_NAME,
Event::MESSAGING_QUEUE_NAME,
Event::MIGRATIONS_QUEUE_NAME,
Event::HAMSTER_QUEUE_NAME
]), 'Queue name')
->param('type', '', new WhiteList([
'success',
'failed',
'processing',
]), 'Queue type')
->inject('queue')
->callback(fn ($name, $type, $queue) => $this->action($name, $type, $queue));
}
/**
* @param string $name The name of the queue to count the jobs from
* @param string $type The type of jobs to count
* @param Connection $queue
*/
public function action(string $name, string $type, Connection $queue): void
{
if (!$name) {
Console::error('Missing required parameter $name');
return;
}
$queueClient = new Client($name, $queue);
$count = 0;
switch ($type) {
case 'success':
$count = $queueClient->countSuccessfulJobs();
break;
case 'failed':
$count = $queueClient->countFailedJobs();
break;
case 'processing':
$count = $queueClient->countProcessingJobs();
break;
};
Console::log("Queue: '{$name}' has {$count} {$type} jobs.");
}
}

64
src/Appwrite/Platform/Tasks/QueueRetry.php Normal file
View file

@ -0,0 +1,64 @@
<?php
namespace Appwrite\Platform\Tasks;
use Appwrite\Event\Event;
use Utopia\CLI\Console;
use Utopia\Platform\Action;
use Utopia\Queue\Client;
use Utopia\Queue\Connection;
use Utopia\Validator\WhiteList;
class QueueRetry extends Action
{
public static function getName(): string
{
return 'queue-retry';
}
public function __construct()
{
$this
->desc('Retry failed jobs from a specific queue identified by the name parameter')
->param('name', '', new WhiteList([
Event::DATABASE_QUEUE_NAME,
Event::DELETE_QUEUE_NAME,
Event::AUDITS_QUEUE_NAME,
Event::MAILS_QUEUE_NAME,
Event::FUNCTIONS_QUEUE_NAME,
Event::USAGE_QUEUE_NAME,
Event::WEBHOOK_CLASS_NAME,
Event::CERTIFICATES_QUEUE_NAME,
Event::BUILDS_QUEUE_NAME,
Event::MESSAGING_QUEUE_NAME,
Event::MIGRATIONS_QUEUE_NAME,
Event::HAMSTER_CLASS_NAME
]), 'Queue name')
->inject('queue')
->callback(fn ($name, $queue) => $this->action($name, $queue));
}
/**
* @param string $name The name of the queue to retry jobs from
* @param Connection $queue
*/
public function action(string $name, Connection $queue): void
{
if (!$name) {
Console::error('Missing required parameter $name');
return;
}
$queueClient = new Client($name, $queue);
if ($queueClient->countFailedJobs() === 0) {
Console::error('No failed jobs found.');
return;
}
Console::log('Retrying failed jobs...');
$queueClient->retry();
}
}

10
src/Appwrite/Platform/Tasks/SSL.php
View file

@ -7,6 +7,7 @@ use Appwrite\Event\Certificate;
use Utopia\App;
use Utopia\CLI\Console;
use Utopia\Database\Document;
use Utopia\Validator\Boolean;
use Utopia\Validator\Hostname;
class SSL extends Action
@ -21,19 +22,22 @@ class SSL extends Action
$this
->desc('Validate server certificates')
->param('domain', App::getEnv('_APP_DOMAIN', ''), new Hostname(), 'Domain to generate certificate for. If empty, main domain will be used.', true)
->param('skip-check', true, new Boolean(true), 'If DNS and renew check should be skipped. Defaults to true, and when true, all jobs will result in certificate generation attempt.', true)
->inject('queueForCertificates')
->callback(fn (string $domain, Certificate $queueForCertificates) => $this->action($domain, $queueForCertificates));
->callback(fn (string $domain, bool|string $skipCheck, Certificate $queueForCertificates) => $this->action($domain, $skipCheck, $queueForCertificates));
}
public function action(string $domain, Certificate $queueForCertificates): void
public function action(string $domain, bool|string $skipCheck, Certificate $queueForCertificates): void
{
$skipCheck = \strval($skipCheck) === 'true';
Console::success('Scheduling a job to issue a TLS certificate for domain: ' . $domain);
$queueForCertificates
->setDomain(new Document([
'domain' => $domain
]))
->setSkipRenewCheck(true)
->setSkipRenewCheck($skipCheck)
->trigger();
}
}

30
src/Appwrite/Platform/Workers/Certificates.php
View file

@ -23,6 +23,7 @@ use Utopia\Database\Helpers\ID;
use Utopia\Database\Query;
use Utopia\Domains\Domain;
use Utopia\Locale\Locale;
use Utopia\Logger\Log;
use Utopia\Platform\Action;
use Utopia\Queue\Message;
@ -45,7 +46,8 @@ class Certificates extends Action
->inject('queueForMails')
->inject('queueForEvents')
->inject('queueForFunctions')
->callback(fn(Message $message, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions) => $this->action($message, $dbForConsole, $queueForMails, $queueForEvents, $queueForFunctions));
->inject('log')
->callback(fn(Message $message, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log) => $this->action($message, $dbForConsole, $queueForMails, $queueForEvents, $queueForFunctions, $log));
}
/**
@ -58,7 +60,7 @@ class Certificates extends Action
* @throws Throwable
* @throws \Utopia\Database\Exception
*/
public function action(Message $message, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions): void
public function action(Message $message, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log): void
{
$payload = $message->getPayload() ?? [];
@ -70,7 +72,7 @@ class Certificates extends Action
$domain = new Domain($document->getAttribute('domain', ''));
$skipRenewCheck = $payload['skipRenewCheck'] ?? false;
$this->execute($domain, $dbForConsole, $queueForMails, $queueForEvents, $queueForFunctions, $skipRenewCheck);
$this->execute($domain, $dbForConsole, $queueForMails, $queueForEvents, $queueForFunctions, $log, $skipRenewCheck);
}
/**
@ -84,7 +86,7 @@ class Certificates extends Action
* @throws Throwable
* @throws \Utopia\Database\Exception
*/
private function execute(Domain $domain, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, bool $skipRenewCheck = false): void
private function execute(Domain $domain, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log, bool $skipRenewCheck = false): void
{
/**
* 1. Read arguments and validate domain
@ -138,11 +140,11 @@ class Certificates extends Action
if (!$skipRenewCheck) {
$mainDomain = $this->getMainDomain();
$isMainDomain = !isset($mainDomain) || $domain->get() === $mainDomain;
$this->validateDomain($domain, $isMainDomain);
$this->validateDomain($domain, $isMainDomain, $log);
}
// If certificate exists already, double-check expiry date. Skip if job is forced
if (!$skipRenewCheck && !$this->isRenewRequired($domain->get())) {
if (!$skipRenewCheck && !$this->isRenewRequired($domain->get(), $log)) {
throw new Exception('Renew isn\'t required.');
}
@ -180,6 +182,8 @@ class Certificates extends Action
// Send email to security email
$this->notifyError($domain->get(), $e->getMessage(), $attempts, $queueForMails);
throw $e;
} finally {
// All actions result in new updatedAt date
$certificate->setAttribute('updated', DateTime::now());
@ -247,7 +251,7 @@ class Certificates extends Action
* @return void
* @throws Exception
*/
private function validateDomain(Domain $domain, bool $isMainDomain): void
private function validateDomain(Domain $domain, bool $isMainDomain, Log $log): void
{
if (empty($domain->get())) {
throw new Exception('Missing certificate domain.');
@ -267,8 +271,15 @@ class Certificates extends Action
}
// Verify domain with DNS records
$validationStart = \microtime(true);
$validator = new CNAME($target->get());
if (!$validator->isValid($domain->get())) {
$log->addExtra('dnsTiming', \strval(\microtime(true) - $validationStart));
$log->addTag('dnsDomain', $domain->get());
$error = $validator->getLogs();
$log->addExtra('dnsResponse', \is_array($error) ? \json_encode($error) : \strval($error));
throw new Exception('Failed to verify domain DNS records.');
}
} else {
@ -284,7 +295,7 @@ class Certificates extends Action
* @return bool True, if certificate needs to be renewed
* @throws Exception
*/
private function isRenewRequired(string $domain): bool
private function isRenewRequired(string $domain, Log $log): bool
{
$certPath = APP_STORAGE_CERTIFICATES . '/' . $domain . '/cert.pem';
if (\file_exists($certPath)) {
@ -294,12 +305,15 @@ class Certificates extends Action
$validTo = $certData['validTo_time_t'] ?? 0;
if (empty($validTo)) {
$log->addTag('certificateDomain', $domain);
throw new Exception('Unable to read certificate file (cert.pem).');
}
// LetsEncrypt allows renewal 30 days before expiry
$expiryInAdvance = (60 * 60 * 24 * 30);
if ($validTo - $expiryInAdvance > \time()) {
$log->addTag('certificateDomain', $domain);
$log->addExtra('certificateData', \is_array($certData) ? \json_encode($certData) : \strval($certData));
return false;
}
}

7
src/Appwrite/Platform/Workers/Deletes.php
View file

@ -517,12 +517,7 @@ class Deletes extends Action
$teamId = $document->getAttribute('teamId');
$team = $dbForProject->getDocument('teams', $teamId);
if (!$team->isEmpty()) {
$team = $dbForProject->updateDocument(
'teams',
$teamId,
// Ensure that total >= 0
$team->setAttribute('total', \max($team->getAttribute('total', 0) - 1, 0))
);
$dbForProject->decreaseDocumentAttribute('teams', $teamId, 'total', 1, 0);
}
}
});

29
src/Appwrite/Platform/Workers/Messaging.php
View file

@ -5,6 +5,7 @@ namespace Appwrite\Platform\Workers;
use Exception;
use Utopia\App;
use Utopia\CLI\Console;
use Utopia\Database\Document;
use Utopia\DSN\DSN;
use Utopia\Messaging\Messages\SMS;
use Utopia\Messaging\Adapters\SMS\Mock;
@ -15,6 +16,7 @@ use Utopia\Messaging\Adapters\SMS\Twilio;
use Utopia\Messaging\Adapters\SMS\Vonage;
use Utopia\Platform\Action;
use Utopia\Queue\Message;
use Appwrite\Event\Usage;
class Messaging extends Action
{
@ -43,20 +45,36 @@ class Messaging extends Action
$this
->desc('Messaging worker')
->inject('message')
->callback(fn($message) => $this->action($message));
->inject('queueForUsage')
->callback(fn(Message $message, Usage $queueForUsage) => $this->action($message, $queueForUsage));
}
/**
* @param Message $message
* @param Usage $queueForUsage
* @return void
* @throws Exception
*/
public function action(Message $message): void
public function action(Message $message, Usage $queueForUsage): void
{
$payload = $message->getPayload() ?? [];
if (empty($payload)) {
Console::error('Payload arg not found');
throw new Exception('Missing payload');
}
if (empty($payload['project'])) {
throw new Exception('Project not set in payload');
}
$project = new Document($payload['project'] ?? []);
Console::log('Project: ' . $project->getId());
$denyList = App::getEnv('_APP_SMS_PROJECTS_DENY_LIST', '');
$denyList = explode(',', $denyList);
if (in_array($project->getId(), $denyList)) {
Console::error("Project is in the deny list. Skipping ...");
return;
}
@ -100,6 +118,11 @@ class Messaging extends Action
try {
$sms->send($message);
$queueForUsage
->setProject($project)
->addMetric(METRIC_MESSAGES, 1)
->trigger();
} catch (\Exception $error) {
throw new Exception('Error sending message: ' . $error->getMessage(), 500);
}

3
src/Appwrite/Utopia/Response.php
View file

@ -70,6 +70,7 @@ use Appwrite\Utopia\Response\Model\Token;
use Appwrite\Utopia\Response\Model\Webhook;
use Appwrite\Utopia\Response\Model\Preferences;
use Appwrite\Utopia\Response\Model\HealthAntivirus;
use Appwrite\Utopia\Response\Model\HealthCertificate;
use Appwrite\Utopia\Response\Model\HealthQueue;
use Appwrite\Utopia\Response\Model\HealthStatus;
use Appwrite\Utopia\Response\Model\HealthTime;
@ -253,6 +254,7 @@ class Response extends SwooleResponse
public const MODEL_HEALTH_QUEUE = 'healthQueue';
public const MODEL_HEALTH_TIME = 'healthTime';
public const MODEL_HEALTH_ANTIVIRUS = 'healthAntivirus';
public const MODEL_HEALTH_CERTIFICATE = 'healthCertificate';
public const MODEL_HEALTH_STATUS_LIST = 'healthStatusList';
// Console
@ -390,6 +392,7 @@ class Response extends SwooleResponse
->setModel(new HealthAntivirus())
->setModel(new HealthQueue())
->setModel(new HealthStatus())
->setModel(new HealthCertificate())
->setModel(new HealthTime())
->setModel(new HealthVersion())
->setModel(new Metric())

71
src/Appwrite/Utopia/Response/Model/HealthCertificate.php Normal file
View file

@ -0,0 +1,71 @@
<?php
namespace Appwrite\Utopia\Response\Model;
use Appwrite\Utopia\Response;
use Appwrite\Utopia\Response\Model;
class HealthCertificate extends Model
{
public function __construct()
{
$this
->addRule('name', [
'type' => self::TYPE_STRING,
'description' => 'Certificate name',
'default' => '',
'example' => '/CN=www.google.com',
])
->addRule('subjectSN', [
'type' => self::TYPE_STRING,
'description' => 'Subject SN',
'default' => 'www.google.com',
'example' => '',
])
->addRule('issuerOrganisation', [
'type' => self::TYPE_STRING,
'description' => 'Issuer organisation',
'default' => 'Google Trust Services LLC',
'example' => '',
])
->addRule('validFrom', [
'type' => self::TYPE_STRING,
'description' => 'Valid from',
'default' => '',
'example' => '1704200998',
])
->addRule('validTo', [
'type' => self::TYPE_STRING,
'description' => 'Valid to',
'default' => '',
'example' => '1711458597',
])
->addRule('signatureTypeSN', [
'type' => self::TYPE_STRING,
'description' => 'Signature type SN',
'default' => '',
'example' => 'RSA-SHA256',
])
;
}
/**
* Get Name
*
* @return string
*/
public function getName(): string
{
return 'Health Certificate';
}
/**
* Get Type
*
* @return string
*/
public function getType(): string
{
return Response::MODEL_HEALTH_CERTIFICATE;
}
}

2
tests/e2e/Services/Databases/DatabasesBase.php
View file

@ -978,7 +978,7 @@ trait DatabasesBase
]);
$this->assertEquals(400, $badEnum['headers']['status-code']);
$this->assertEquals('Invalid `elements` param: Value must a valid array and Value must be a valid string and at least 1 chars and no longer than 255 chars', $badEnum['body']['message']);
$this->assertEquals('Invalid `elements` param: Value must a valid array no longer than 100 items and Value must be a valid string and at least 1 chars and no longer than 255 chars', $badEnum['body']['message']);
return $data;
}

70
tests/e2e/Services/Health/HealthCustomServerTest.php
View file

@ -424,4 +424,74 @@ class HealthCustomServerTest extends Scope
return [];
}
public function testCertificateValidity(): array
{
/**
* Test for SUCCESS
*/
$response = $this->client->call(Client::METHOD_GET, '/health/certificate?domain=www.google.com', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), []);
$this->assertEquals(200, $response['headers']['status-code']);
$this->assertEquals('/CN=www.google.com', $response['body']['name']);
$this->assertEquals('www.google.com', $response['body']['subjectSN']);
$this->assertEquals('Google Trust Services LLC', $response['body']['issuerOrganisation']);
$this->assertIsInt($response['body']['validFrom']);
$this->assertIsInt($response['body']['validTo']);
$response = $this->client->call(Client::METHOD_GET, '/health/certificate?domain=appwrite.io', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), []);
$this->assertEquals(200, $response['headers']['status-code']);
$this->assertEquals('/CN=appwrite.io', $response['body']['name']);
$this->assertEquals('appwrite.io', $response['body']['subjectSN']);
$this->assertEquals("Let's Encrypt", $response['body']['issuerOrganisation']);
$this->assertIsInt($response['body']['validFrom']);
$this->assertIsInt($response['body']['validTo']);
$response = $this->client->call(Client::METHOD_GET, '/health/certificate?domain=https://google.com', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), []);
$this->assertEquals(200, $response['headers']['status-code']);
/**
* Test for FAILURE
*/
$response = $this->client->call(Client::METHOD_GET, '/health/certificate?domain=localhost', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), []);
$this->assertEquals(400, $response['headers']['status-code']);
$response = $this->client->call(Client::METHOD_GET, '/health/certificate?domain=doesnotexist.com', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), []);
$this->assertEquals(404, $response['headers']['status-code']);
$response = $this->client->call(Client::METHOD_GET, '/health/certificate?domain=www.google.com/usr/src/local', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), []);
$this->assertEquals(400, $response['headers']['status-code']);
$response = $this->client->call(Client::METHOD_GET, '/health/certificate?domain=', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), []);
$this->assertEquals(400, $response['headers']['status-code']);
return [];
}
}