From f0bb300bd63c48f20747ef4eb48e0d539319029e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Thu, 12 May 2022 10:31:42 +0000 Subject: [PATCH 1/9] Import fix --- app/workers/certificates.php | 1 + 1 file changed, 1 insertion(+) diff --git a/app/workers/certificates.php b/app/workers/certificates.php index f1df74972c..d07a678f98 100644 --- a/app/workers/certificates.php +++ b/app/workers/certificates.php @@ -5,6 +5,7 @@ use Appwrite\Network\Validator\CNAME; use Appwrite\Resque\Worker; use Utopia\App; use Utopia\CLI\Console; +use Utopia\Database\Database; use Utopia\Database\Document; use Utopia\Database\Query; use Utopia\Database\Validator\Authorization; From 1c2fdab9eaaec050e91150925897a5c79c5c4d46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Thu, 12 May 2022 10:38:48 +0000 Subject: [PATCH 2/9] Bug fix --- app/workers/certificates.php | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/app/workers/certificates.php b/app/workers/certificates.php index d07a678f98..7a4719aec9 100644 --- a/app/workers/certificates.php +++ b/app/workers/certificates.php @@ -397,7 +397,10 @@ class CertificatesV1 extends Worker $domainDocument->setAttribute('certificateId', $certificateId); $this->dbForConsole->updateDocument('domains', $domainDocument->getId(), $domainDocument); - $this->dbForConsole->deleteCachedDocument('projects', $domainDocument->getAttribute('projectId')); + + if($domainDocument->getAttribute('projectId')) { + $this->dbForConsole->deleteCachedDocument('projects', $domainDocument->getAttribute('projectId')); + } } } } From 3efb90415177bb89ff400cfdce27461904032a5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Thu, 12 May 2022 10:56:25 +0000 Subject: [PATCH 3/9] Prevent unnecessary SSL jobs --- app/controllers/general.php | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/app/controllers/general.php b/app/controllers/general.php index 540c087dae..9bc1d789ee 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -99,13 +99,13 @@ App::init(function ($utopia, $request, $response, $console, $project, $dbForCons ]); $domainDocument = $dbForConsole->createDocument('domains', $domainDocument); - } - Console::info('Issuing a TLS certificate for the main domain (' . $domain->get() . ') in a few seconds...'); - - Resque::enqueue(Event::CERTIFICATES_QUEUE_NAME, Event::CERTIFICATES_CLASS_NAME, [ - 'domain' => $domain->get() - ]); + Console::info('Issuing a TLS certificate for the main domain (' . $domain->get() . ') in a few seconds...'); + + Resque::enqueue(Event::CERTIFICATES_QUEUE_NAME, Event::CERTIFICATES_CLASS_NAME, [ + 'domain' => $domain->get() + ]); + } } $domains[$domain->get()] = true; From 55d934381e27c3a5f83f16b486db14facb23b4d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Thu, 12 May 2022 11:40:32 +0000 Subject: [PATCH 4/9] Fix Traefik config file text --- app/workers/certificates.php | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/app/workers/certificates.php b/app/workers/certificates.php index 7a4719aec9..d8d5a98707 100644 --- a/app/workers/certificates.php +++ b/app/workers/certificates.php @@ -334,11 +334,12 @@ class CertificatesV1 extends Worker throw new Exception('Failed to rename certificate privkey.pem.'); } - $config = - "tls:" . - " certificates:" . - " - certFile: /storage/certificates/{$domain}/fullchain.pem" . - " keyFile: /storage/certificates/{$domain}/privkey.pem"; + $config = \implode('\n', [ + "tls:", + " certificates:", + " - certFile: /storage/certificates/{$domain}/fullchain.pem", + " keyFile: /storage/certificates/{$domain}/privkey.pem" + ]); // Save configuration into Traefik using our new cert files if (!\file_put_contents(APP_STORAGE_CONFIG . '/' . $domain . '.yml', $config)) { From 417502338a23be0cbfa9f53903368d04355fffaa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Thu, 12 May 2022 11:50:56 +0000 Subject: [PATCH 5/9] Proper line break symbol --- app/workers/certificates.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/workers/certificates.php b/app/workers/certificates.php index d8d5a98707..9d4f8ac4e9 100644 --- a/app/workers/certificates.php +++ b/app/workers/certificates.php @@ -334,7 +334,7 @@ class CertificatesV1 extends Worker throw new Exception('Failed to rename certificate privkey.pem.'); } - $config = \implode('\n', [ + $config = \implode(PHP_EOL, [ "tls:", " certificates:", " - certFile: /storage/certificates/{$domain}/fullchain.pem", From a1e0390ae77554fe4284c4cff3b9d37ef4ce86b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Thu, 12 May 2022 12:08:50 +0000 Subject: [PATCH 6/9] Imrpove var names, logs --- app/config/locale/translations/en.json | 4 ++-- app/tasks/ssl.php | 2 +- app/workers/certificates.php | 31 ++++++++++++++------------ 3 files changed, 20 insertions(+), 17 deletions(-) diff --git a/app/config/locale/translations/en.json b/app/config/locale/translations/en.json index e533d82795..ab765fe47e 100644 --- a/app/config/locale/translations/en.json +++ b/app/config/locale/translations/en.json @@ -29,8 +29,8 @@ "emails.invitation.signature": "{{project}} team", "emails.certificate.subject": "Certificate failure for %s", "emails.certificate.hello": "Hello", - "emails.certificate.body": "Certificate for your domain '{{domain}}' could not be renewed. This is attempt no. {{attempt}}, and the failure was caused by: {{error}}", - "emails.certificate.footer": "Certificate will still be valid for 30 days since first failure. We highly recommend investigating the case, otherwise your domain will end up without a secure certificate.", + "emails.certificate.body": "Certificate for your domain '{{domain}}' could not be generated. This is attempt no. {{attempt}}, and the failure was caused by: {{error}}", + "emails.certificate.footer": "Your previous certificate willl be valid for 30 days since the first failure. We highly recommend investigating this case, otherwise your domain will end up without a valid SSL communication.", "emails.certificate.thanks": "Thanks", "emails.certificate.signature": "{{project}} team", "locale.country.unknown": "Unknown", diff --git a/app/tasks/ssl.php b/app/tasks/ssl.php index f28c0b8532..b6dae6f9f7 100644 --- a/app/tasks/ssl.php +++ b/app/tasks/ssl.php @@ -17,6 +17,6 @@ $cli // Scheduje a job Resque::enqueue(Event::CERTIFICATES_QUEUE_NAME, Event::CERTIFICATES_CLASS_NAME, [ 'domain' => $domain, - 'skipRenewCheck' => true + 'skipCheck' => true ]); }); \ No newline at end of file diff --git a/app/workers/certificates.php b/app/workers/certificates.php index 9d4f8ac4e9..502d4498ec 100644 --- a/app/workers/certificates.php +++ b/app/workers/certificates.php @@ -74,7 +74,7 @@ class CertificatesV1 extends Worker try { // Read arguments $domain = $this->args['domain']; // String of domain (hostname) - $skipRenewCheck = $this->args['skipRenewCheck'] ?? false; // If true, we won't double-check expiry from cert file + $skipCheck = $this->args['skipCheck'] ?? false; // If true, we won't double-check expiry from cert file $domain = new Domain((!empty($domain)) ? $domain : ''); @@ -93,13 +93,15 @@ class CertificatesV1 extends Worker throw new Exception('You must set a valid security email address (_APP_SYSTEM_SECURITY_EMAIL_ADDRESS) to issue an SSL certificate.'); } - $mainDomain = $this->getMainDomain(); - $isMainDomain = !isset($mainDomain) || $domain->get() === $mainDomain; - $this->validateDomain($domain, $isMainDomain); + // Validate domain and DNS records. Skip if job is forced + if(!$skipCheck) { + $mainDomain = $this->getMainDomain(); + $isMainDomain = !isset($mainDomain) || $domain->get() === $mainDomain; + $this->validateDomain($domain, $isMainDomain); + } - // If certificate exists already, double-check expiry date - // If asked to skip, we won't - if(!$skipRenewCheck && !$this->isRenewRequired($domain->get())) { + // If certificate exists already, double-check expiry date. Skip if job is forced + if(!$skipCheck && !$this->isRenewRequired($domain->get())) { throw new Exception('Renew isn\'t required.'); } @@ -114,7 +116,7 @@ class CertificatesV1 extends Worker ])); // Give certificates to Traefik - $this->applyCertificateFiles($domain->get()); + $this->applyCertificateFiles($domain->get(), $letsEncryptData); // Update certificate info stored in database $certificate->setAttribute('renewDate', $this->getRenewDate($domain->get())); @@ -305,10 +307,11 @@ class CertificatesV1 extends Worker * Method to take files from Let's Encrypt, and put it into Traefik. * * @param string $domain Domain which certificate was generated for + * @param array $letsEncryptData Let's Encrypt logs to use for additional info when throwing error * * @return void */ - private function applyCertificateFiles(string $domain): void { + private function applyCertificateFiles(string $domain, array $letsEncryptData): void { // Prepare folder in storage for domain $path = APP_STORAGE_CERTIFICATES . '/' . $domain; if (!\is_readable($path)) { @@ -318,20 +321,20 @@ class CertificatesV1 extends Worker } // Move generated files from certbot into our storage - if(!@\rename('/etc/letsencrypt/live/'.$domain.'/cert.pem', APP_STORAGE_CERTIFICATES.'/'.$domain.'/cert.pem')) { - throw new Exception('Failed to rename certificate cert.pem.'); + if(!@\rename('/etc/letsencrypt/live/'.$domain.'/cert.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/cert.pem')) { + throw new Exception('Failed to rename certificate cert.pem. Let\'s Encrypt log: ' . $letsEncryptData['stderr']); } if (!@\rename('/etc/letsencrypt/live/' . $domain . '/chain.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/chain.pem')) { - throw new Exception('Failed to rename certificate chain.pem.'); + throw new Exception('Failed to rename certificate chain.pem. Let\'s Encrypt log: ' . $letsEncryptData['stderr']); } if (!@\rename('/etc/letsencrypt/live/' . $domain . '/fullchain.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/fullchain.pem')) { - throw new Exception('Failed to rename certificate fullchain.pem.'); + throw new Exception('Failed to rename certificate fullchain.pem. Let\'s Encrypt log: ' . $letsEncryptData['stderr']); } if (!@\rename('/etc/letsencrypt/live/' . $domain . '/privkey.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/privkey.pem')) { - throw new Exception('Failed to rename certificate privkey.pem.'); + throw new Exception('Failed to rename certificate privkey.pem. Let\'s Encrypt log: ' . $letsEncryptData['stderr']); } $config = \implode(PHP_EOL, [ From 762c1cfe730666542b0c7c79b9a87c8b5cd72784 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Thu, 12 May 2022 12:13:47 +0000 Subject: [PATCH 7/9] Improve logs --- app/tasks/ssl.php | 2 +- app/workers/certificates.php | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/app/tasks/ssl.php b/app/tasks/ssl.php index b6dae6f9f7..743be92a96 100644 --- a/app/tasks/ssl.php +++ b/app/tasks/ssl.php @@ -12,7 +12,7 @@ $cli ->desc('Validate server certificates') ->param('domain', App::getEnv('_APP_DOMAIN', ''), new Hostname(), 'Domain to generate certificate for. If empty, main domain will be used.', true) ->action(function ($domain) { - Console::success('Scheduling a job to issue a TLS certificate for domain:' . $domain); + Console::success('Scheduling a job to issue a TLS certificate for domain: ' . $domain); // Scheduje a job Resque::enqueue(Event::CERTIFICATES_QUEUE_NAME, Event::CERTIFICATES_CLASS_NAME, [ diff --git a/app/workers/certificates.php b/app/workers/certificates.php index 502d4498ec..c52c93a540 100644 --- a/app/workers/certificates.php +++ b/app/workers/certificates.php @@ -322,19 +322,19 @@ class CertificatesV1 extends Worker // Move generated files from certbot into our storage if(!@\rename('/etc/letsencrypt/live/'.$domain.'/cert.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/cert.pem')) { - throw new Exception('Failed to rename certificate cert.pem. Let\'s Encrypt log: ' . $letsEncryptData['stderr']); + throw new Exception('Failed to rename certificate cert.pem. Let\'s Encrypt log: ' . $letsEncryptData['stderr'] . '; ' . $letsEncryptData['stdout']); } if (!@\rename('/etc/letsencrypt/live/' . $domain . '/chain.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/chain.pem')) { - throw new Exception('Failed to rename certificate chain.pem. Let\'s Encrypt log: ' . $letsEncryptData['stderr']); + throw new Exception('Failed to rename certificate chain.pem. Let\'s Encrypt log: ' . $letsEncryptData['stderr'] . '; ' . $letsEncryptData['stdout']); } if (!@\rename('/etc/letsencrypt/live/' . $domain . '/fullchain.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/fullchain.pem')) { - throw new Exception('Failed to rename certificate fullchain.pem. Let\'s Encrypt log: ' . $letsEncryptData['stderr']); + throw new Exception('Failed to rename certificate fullchain.pem. Let\'s Encrypt log: ' . $letsEncryptData['stderr'] . '; ' . $letsEncryptData['stdout']); } if (!@\rename('/etc/letsencrypt/live/' . $domain . '/privkey.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/privkey.pem')) { - throw new Exception('Failed to rename certificate privkey.pem. Let\'s Encrypt log: ' . $letsEncryptData['stderr']); + throw new Exception('Failed to rename certificate privkey.pem. Let\'s Encrypt log: ' . $letsEncryptData['stderr'] . '; ' . $letsEncryptData['stdout']); } $config = \implode(PHP_EOL, [ From 88813296cf4ad5a232ba48cd57bc04c7ecb3a9b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Thu, 12 May 2022 12:14:59 +0000 Subject: [PATCH 8/9] Verbose log improvement --- app/workers/certificates.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/workers/certificates.php b/app/workers/certificates.php index c52c93a540..f932e4b8f8 100644 --- a/app/workers/certificates.php +++ b/app/workers/certificates.php @@ -322,19 +322,19 @@ class CertificatesV1 extends Worker // Move generated files from certbot into our storage if(!@\rename('/etc/letsencrypt/live/'.$domain.'/cert.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/cert.pem')) { - throw new Exception('Failed to rename certificate cert.pem. Let\'s Encrypt log: ' . $letsEncryptData['stderr'] . '; ' . $letsEncryptData['stdout']); + throw new Exception('Failed to rename certificate cert.pem. Let\'s Encrypt log: ' . $letsEncryptData['stderr'] . ' ; ' . $letsEncryptData['stdout']); } if (!@\rename('/etc/letsencrypt/live/' . $domain . '/chain.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/chain.pem')) { - throw new Exception('Failed to rename certificate chain.pem. Let\'s Encrypt log: ' . $letsEncryptData['stderr'] . '; ' . $letsEncryptData['stdout']); + throw new Exception('Failed to rename certificate chain.pem. Let\'s Encrypt log: ' . $letsEncryptData['stderr'] . ' ; ' . $letsEncryptData['stdout']); } if (!@\rename('/etc/letsencrypt/live/' . $domain . '/fullchain.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/fullchain.pem')) { - throw new Exception('Failed to rename certificate fullchain.pem. Let\'s Encrypt log: ' . $letsEncryptData['stderr'] . '; ' . $letsEncryptData['stdout']); + throw new Exception('Failed to rename certificate fullchain.pem. Let\'s Encrypt log: ' . $letsEncryptData['stderr'] . ' ; ' . $letsEncryptData['stdout']); } if (!@\rename('/etc/letsencrypt/live/' . $domain . '/privkey.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/privkey.pem')) { - throw new Exception('Failed to rename certificate privkey.pem. Let\'s Encrypt log: ' . $letsEncryptData['stderr'] . '; ' . $letsEncryptData['stdout']); + throw new Exception('Failed to rename certificate privkey.pem. Let\'s Encrypt log: ' . $letsEncryptData['stderr'] . ' ; ' . $letsEncryptData['stdout']); } $config = \implode(PHP_EOL, [ From 613fee0fee658084fcda1a5164397041e556578f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Thu, 12 May 2022 12:32:13 +0000 Subject: [PATCH 9/9] Empty commit for CI/CD