From 0a64d46d4c1b7b7330cd4421dffad1ab244d0eb2 Mon Sep 17 00:00:00 2001 From: Riddhesh Sanghvi Date: Tue, 30 Jun 2026 18:00:58 +0530 Subject: [PATCH 1/4] feat(ssl): add inter-site jitter and clear rate-limit messaging to ssl-renew --- src/Site_Command.php | 8 +++++++ src/helper/Site_Letsencrypt.php | 42 ++++++++++++++++++++++++++++++--- 2 files changed, 47 insertions(+), 3 deletions(-) diff --git a/src/Site_Command.php b/src/Site_Command.php index 9967fdcf..4144532f 100644 --- a/src/Site_Command.php +++ b/src/Site_Command.php @@ -104,6 +104,8 @@ public function __invoke( $args, $assoc_args ) { } elseif ( in_array( reset( $args ), [ 'ssl-renew' ], true ) && array_key_exists( 'all', $assoc_args ) ) { $sites = Site::all(); unset( $assoc_args['all'] ); + // Spread per-site dispatches over time to avoid bursting against Let's Encrypt rate limits. + $dispatched = false; foreach ( $sites as $site ) { $type = $site->site_type; $args = [ 'site', 'ssl-renew', $site->site_url ]; @@ -125,11 +127,17 @@ public function __invoke( $args, $assoc_args ) { continue; } + // Jitter between sites only (not before the first / after the last) to smooth burst load on the LE API. + if ( $dispatched ) { + sleep( random_int( 1, 5 ) ); + } + $command = EE::get_root_command(); $leaf_command = CommandFactory::create( 'site', $callback, $command ); $command->add_subcommand( 'site', $leaf_command ); EE::run_command( $args, $assoc_args ); + $dispatched = true; } die; } else { diff --git a/src/helper/Site_Letsencrypt.php b/src/helper/Site_Letsencrypt.php index 0669eaf1..76bcbf47 100644 --- a/src/helper/Site_Letsencrypt.php +++ b/src/helper/Site_Letsencrypt.php @@ -16,6 +16,7 @@ use AcmePhp\Core\Challenge\WaitingValidator; use AcmePhp\Core\Exception\Protocol\ChallengeNotSupportedException; use AcmePhp\Core\Exception\Protocol\CertificateRevocationException; +use AcmePhp\Core\Exception\Server\RateLimitedServerException; use AcmePhp\Core\Protocol\AuthorizationChallenge; use AcmePhp\Core\Protocol\ResourcesDirectory; use AcmePhp\Core\Protocol\RevocationReason; @@ -208,7 +209,12 @@ public function authorize( Array $domains, $wildcard = false, $preferred_challen try { $order = $this->client->requestOrder( $domains ); } catch ( \Exception $e ) { - \EE::warning( 'It seems you\'re in local environment or using non-public domain, please check logs. Skipping letsencrypt.' ); + // A rate-limit is a distinct failure from a non-public domain; emit a clear, actionable message for it. + if ( $this->is_rate_limit_exception( $e ) ) { + \EE::warning( 'Let\'s Encrypt rate limit hit for: ' . implode( ', ', $domains ) . '. Please wait before retrying. Ref: https://letsencrypt.org/docs/rate-limits/' ); + } else { + \EE::warning( 'It seems you\'re in local environment or using non-public domain, please check logs. Skipping letsencrypt.' ); + } \EE::log( 'You can fix the issue and re-run: ee site ssl-verify ' . $domains[0] ); return false; @@ -568,6 +574,26 @@ public function isRenewalNecessary( $domain ) { return true; } + /** + * Whether the given exception represents a Let's Encrypt rate-limit response. + * + * Matches the acmephp RateLimitedServerException as well as the 'rateLimited' ACME error + * type / HTTP 429 surfaced in the message, so callers can show rate-limit-specific guidance. + * + * @param \Throwable $e + * + * @return bool + */ + private function is_rate_limit_exception( $e ) { + if ( $e instanceof RateLimitedServerException ) { + return true; + } + + $message = strtolower( $e->getMessage() ); + + return ( false !== strpos( $message, 'ratelimited' ) || false !== strpos( $message, 'too many' ) ); + } + /** * Renew a given domain certificate. * @@ -644,7 +670,12 @@ private function executeRenewal( $domain, array $alternativeNames, $force = fals \EE::warning( 'A critical error occured during certificate renewal' ); \EE::debug( print_r( $e, true ) ); - \EE::warning( 'Challenge Authorization failed. Check logs and check if your domain is pointed correctly to this server.' ); + // A rate-limit is not a misconfigured-domain failure; point the user to the LE rate-limit docs instead. + if ( $this->is_rate_limit_exception( $e ) ) { + \EE::warning( 'Let\'s Encrypt rate limit hit for: ' . $domain . '. Please wait before retrying. Ref: https://letsencrypt.org/docs/rate-limits/' ); + } else { + \EE::warning( 'Challenge Authorization failed. Check logs and check if your domain is pointed correctly to this server.' ); + } \EE::log( 'You can fix the issue and re-run: ee site ssl-verify ' . $domains[0] ); return false; @@ -652,7 +683,12 @@ private function executeRenewal( $domain, array $alternativeNames, $force = fals \EE::warning( 'A critical error occured during certificate renewal' ); \EE::debug( print_r( $e, true ) ); - \EE::warning( 'Challenge Authorization failed. Check logs and check if your domain is pointed correctly to this server.' ); + // A rate-limit is not a misconfigured-domain failure; point the user to the LE rate-limit docs instead. + if ( $this->is_rate_limit_exception( $e ) ) { + \EE::warning( 'Let\'s Encrypt rate limit hit for: ' . $domain . '. Please wait before retrying. Ref: https://letsencrypt.org/docs/rate-limits/' ); + } else { + \EE::warning( 'Challenge Authorization failed. Check logs and check if your domain is pointed correctly to this server.' ); + } \EE::log( 'You can fix the issue and re-run: ee site ssl-verify ' . $domains[0] ); return false; From 07145dc240429b6ff24d35769454d5100491b1cc Mon Sep 17 00:00:00 2001 From: Riddhesh Sanghvi Date: Thu, 24 Sep 2026 10:41:07 +0000 Subject: [PATCH 2/4] fix(ssl): report rate limit when revoking stale authorizations hits it revokeAuthorizationChallenges() runs before authorize() on every renewal and also calls new-order, so a Let's Encrypt rate limit surfaced there as an uncaught RateLimitedServerException: the new rate-limit message was never shown and `ssl-renew --all` aborted on that site. Stop revoking on a rate limit and let authorize() report it. --- src/helper/Site_Letsencrypt.php | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/helper/Site_Letsencrypt.php b/src/helper/Site_Letsencrypt.php index 76bcbf47..11162b4d 100644 --- a/src/helper/Site_Letsencrypt.php +++ b/src/helper/Site_Letsencrypt.php @@ -285,6 +285,11 @@ public function revokeAuthorizationChallenges( array $domains ) { \EE::debug( 'Domain Authorization Challenge for ' . $domain . ' revoked successfully' ); } catch ( CertificateRevocationException | AcmeCliException $e ) { \EE::debug( $e->getMessage() ); + } catch ( RateLimitedServerException $e ) { + // Revoking uses new-order too; stop here and let authorize() report the rate limit. + \EE::debug( $e->getMessage() ); + + return; } } else { \EE::debug( 'Domain Authorization Challenge for ' . $domain . ' not found locally' ); From 0318c848d29ac70abc127bb89bddfc96190f2873 Mon Sep 17 00:00:00 2001 From: Riddhesh Sanghvi Date: Thu, 24 Sep 2026 10:41:25 +0000 Subject: [PATCH 3/4] fix(ssl): drop the loose "too many" rate-limit message match The fallback matched any error containing "too many" (e.g. "Too many open files"), which would be reported as a Let's Encrypt rate limit. acmephp maps every `rateLimited` ACME error to RateLimitedServerException, so the class check plus the `rateLimited` marker is enough. Also correct the docblock, which claimed HTTP 429 detection the code never did. --- src/helper/Site_Letsencrypt.php | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src/helper/Site_Letsencrypt.php b/src/helper/Site_Letsencrypt.php index 11162b4d..6b4ec04f 100644 --- a/src/helper/Site_Letsencrypt.php +++ b/src/helper/Site_Letsencrypt.php @@ -580,10 +580,7 @@ public function isRenewalNecessary( $domain ) { } /** - * Whether the given exception represents a Let's Encrypt rate-limit response. - * - * Matches the acmephp RateLimitedServerException as well as the 'rateLimited' ACME error - * type / HTTP 429 surfaced in the message, so callers can show rate-limit-specific guidance. + * Whether the given exception is a Let's Encrypt `rateLimited` ACME error. * * @param \Throwable $e * @@ -594,9 +591,8 @@ private function is_rate_limit_exception( $e ) { return true; } - $message = strtolower( $e->getMessage() ); - - return ( false !== strpos( $message, 'ratelimited' ) || false !== strpos( $message, 'too many' ) ); + // No bare "too many" match: it also hits unrelated errors like "Too many open files". + return false !== stripos( $e->getMessage(), 'ratelimited' ); } /** From 0043a2da2940a60ea19dbfec603e4472394b1f0c Mon Sep 17 00:00:00 2001 From: Riddhesh Sanghvi Date: Thu, 24 Sep 2026 10:42:39 +0000 Subject: [PATCH 4/4] fix(ssl): only jitter between renewals that actually contact Let's Encrypt The jitter slept before every dispatched LE site in `ssl-renew --all`, including the sites whose certificates aren't due, which make no ACME calls. On the daily cron that added about 3s per site for nothing (roughly 5 minutes for 100 sites) while holding up the run. Move it into renew_ssl_cert() after the renewal-necessity check, so it only spaces out real renewals within one process. --- src/Site_Command.php | 8 -------- src/helper/class-ee-site.php | 11 +++++++++++ 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/src/Site_Command.php b/src/Site_Command.php index 4144532f..9967fdcf 100644 --- a/src/Site_Command.php +++ b/src/Site_Command.php @@ -104,8 +104,6 @@ public function __invoke( $args, $assoc_args ) { } elseif ( in_array( reset( $args ), [ 'ssl-renew' ], true ) && array_key_exists( 'all', $assoc_args ) ) { $sites = Site::all(); unset( $assoc_args['all'] ); - // Spread per-site dispatches over time to avoid bursting against Let's Encrypt rate limits. - $dispatched = false; foreach ( $sites as $site ) { $type = $site->site_type; $args = [ 'site', 'ssl-renew', $site->site_url ]; @@ -127,17 +125,11 @@ public function __invoke( $args, $assoc_args ) { continue; } - // Jitter between sites only (not before the first / after the last) to smooth burst load on the LE API. - if ( $dispatched ) { - sleep( random_int( 1, 5 ) ); - } - $command = EE::get_root_command(); $leaf_command = CommandFactory::create( 'site', $callback, $command ); $command->add_subcommand( 'site', $leaf_command ); EE::run_command( $args, $assoc_args ); - $dispatched = true; } die; } else { diff --git a/src/helper/class-ee-site.php b/src/helper/class-ee-site.php index 08ad268e..ef5ff676 100644 --- a/src/helper/class-ee-site.php +++ b/src/helper/class-ee-site.php @@ -65,6 +65,11 @@ abstract class EE_Site_Command { */ public $site_meta; + /** + * @var bool $le_renewal_started Whether this process already started an LE renewal (`ssl-renew --all` renews every site in one process). + */ + private static $le_renewal_started = false; + public function __construct() { $this->fs = new Filesystem(); @@ -2011,6 +2016,12 @@ private function renew_ssl_cert( $args, $force ) { return 0; } + // Space out consecutive renewals to smooth LE API load; sites not due returned above, so they don't wait. + if ( self::$le_renewal_started ) { + sleep( random_int( 1, 5 ) ); + } + self::$le_renewal_started = true; + $postfix_exists = \EE_DOCKER::service_exists( 'postfix', $this->site_data['site_fs_path'] ); $containers_to_start = $postfix_exists ? [ 'nginx', 'postfix' ] : [ 'nginx' ]; $this->www_ssl_wrapper( $containers_to_start, false, $force, true );