1. The problem I’m having:
Migrated Caddy from version 2.6.2 to 2.8.0. Facing issue in zerossl issuance.
Issues:
- Zerossl issuance not working and throwing below error
{“level”:“error”,“ts”:1718622299.338324,“logger”:“tls.renew”,“msg”:“could not get certificate from issuer”,“identifier”:“test21.whizzygeeks.com”,“issuer”:“zerossl”,“error”:“creating certificate: POST https://api.zerossl.com/certificates?access_key=redacted: HTTP 200: API error 2836: csr_cn_is_invalid (details=map) (raw={"success":false,"error":{"code":2836,"type":"csr_cn_is_invalid"}} decode_error=json: unknown field "success")”}
{“level”:“debug”,“ts”:1718622303.069677,“logger”:“events”,“msg”:“event”,“name”:“cert_failed”,“id”:“1bbb8a36-12ba-456b-80ce-feaabcce42ba”,“origin”:“tls”,“data”:{“error”:{},“identifier”:“test21.whizzygeeks.com”,“issuers”:[“zerossl”,“acme-v02.api.letsencrypt.org-directory”],“remaining”:7822501232249134,“renewal”:true}}
- All certificate are being reissued after upgrade from version 2.6.2 to 2.8.0
- Same config is working perfectly in version 2.6.2
Note: Ignore Letsencrypt error rate limit which are usually not visible when zerossl works fine
2. Error messages and/or full log output:
{"level":"debug","ts":1718622113.8828826,"logger":"tls.permission.http","msg":"asking permission endpoint","remote":"","domain":"test21.whizzygeeks.com","url":"http://domain-validator.whizzygeeks.com/px-validate?domain=test21.whizzygeeks.com"}
{"level":"debug","ts":1718622113.886623,"logger":"http.handlers.reverse_proxy","msg":"selected upstream","dial":"localhost:6081","total_upstreams":1}
{"level":"debug","ts":1718622113.8871162,"logger":"http.handlers.reverse_proxy","msg":"upstream roundtrip","upstream":"localhost:6081","duration":0.000416818,"request":{"remote_ip":"54.159.182.238","remote_port":"51364","client_ip":"54.159.182.238","proto":"HTTP/2.0","method":"GET","host":"test21.whizzygeeks.com","uri":"/","headers":{"X-Forwarded-For":["54.159.182.238:51364"],"X-Forwarded-Proto":["https"],"Access-Control-Allow-Origin":["{http.request.access_control_allow_origin}"],"X-Forwarded-Port":[""],"Origin":["{http.request.origin}"],"X-Real-Ip":["54.159.182.238:51364"],"User-Agent":["curl/7.81.0"],"Accept":["*/*"],"X-Forwarded-Host":["test21.whizzygeeks.com"],"Access-Control-Allow-Credentials":["{http.request.access_control_allow_credentials}"],"Access-Control-Max-Age":["{http.request.access_control_max_age}"]},"tls":{"resumed":false,"version":772,"cipher_suite":4865,"proto":"h2","server_name":"test21.whizzygeeks.com"}},"headers":{"X-Cache-Hits":["15"],"Mobile":[""],"Server":["nginx"],"X-Ratelimit-Limit":["300"],"X-Ua":["Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36"],"Age":["1632"],"X-Varnish":["983044 3"],"Host":["test21.whizzygeeks.com"],"Via":["1.1 varnish (Varnish/6.0)"],"X-Cache":["HIT"],"Accept-Ranges":["bytes"],"Content-Type":["text/html; charset=UTF-8"],"X-Ratelimit-Remaining":["299"],"Strict-Transport-Security":["max-age=31536000; includeSubDomains"],"Connection":["keep-alive"],"Date":["Mon, 17 Jun 2024 10:34:41 GMT"],"Vary":["Accept-Encoding"],"X-Content-Type-Options":["nosniff"],"X-Device":["pc"]},"status":200}
{"level":"debug","ts":1718622113.9641237,"logger":"tls.permission.http","msg":"response from permission endpoint","remote":"","domain":"test21.whizzygeeks.com","url":"http://domain-validator.whizzygeeks.com/px-validate?domain=test21.whizzygeeks.com","status":200}
{"level":"info","ts":1718622113.9641638,"logger":"tls.on_demand","msg":"attempting certificate renewal","remote_ip":"54.159.182.238","remote_port":"51364","server_name":"test21.whizzygeeks.com","subjects":["test21.whizzygeeks.com"],"expiration":1726444800,"remaining":7822686.117271079,"revoked":false}
{"level":"info","ts":1718622113.9649742,"logger":"tls.renew","msg":"acquiring lock","identifier":"test21.whizzygeeks.com"}
{"level":"info","ts":1718622113.9651086,"logger":"tls.renew","msg":"lock acquired","identifier":"test21.whizzygeeks.com"}
{"level":"debug","ts":1718622113.9657342,"logger":"tls","msg":"loading managed certificate","domain":"test21.whizzygeeks.com","expiration":1726444800,"issuer_key":"zerossl","storage":"{\"address\":\"primary-redis.whizzygeeks.com:6379\",\"host\":\"primary-redis.whizzygeeks.com\",\"port\":\"6379\",\"db\":6,\"username\":\"\",\"password\":\"REDACTED\",\"timeout\":60,\"key_prefix\":\"caddytls\",\"value_prefix\":\"caddy-storage-redis\",\"aes_key\":\"\",\"tls_enabled\":false,\"tls_insecure\":true}"}
{"level":"info","ts":1718622113.9658303,"logger":"tls.renew","msg":"renewing certificate","identifier":"test21.whizzygeeks.com","remaining":7822686.034173003}
{"level":"debug","ts":1718622113.9658585,"logger":"events","msg":"event","name":"cert_obtaining","id":"e85c7397-ab55-447a-b0ca-40838f336071","origin":"tls","data":{"forced":false,"identifier":"test21.whizzygeeks.com","issuer":"zerossl","remaining":7822686034173003,"renewal":true}}
{"level":"info","ts":1718622113.9660556,"logger":"tls.issuance.zerossl","msg":"creating certificate","identifiers":["test21.whizzygeeks.com"]}
{"level":"error","ts":1718622114.3268423,"logger":"tls.renew","msg":"could not get certificate from issuer","identifier":"test21.whizzygeeks.com","issuer":"zerossl","error":"creating certificate: POST https://api.zerossl.com/certificates?access_key=redacted: HTTP 200: API error 2836: csr_cn_is_invalid (details=map[]) (raw={\"success\":false,\"error\":{\"code\":2836,\"type\":\"csr_cn_is_invalid\"}} decode_error=json: unknown field \"success\")"}
{"level":"info","ts":1718622114.3273354,"logger":"http","msg":"waiting on internal rate limiter","identifiers":["test21.whizzygeeks.com"],"ca":"https://acme-v02.api.letsencrypt.org/directory","account":"admin@admin.com"}
{"level":"info","ts":1718622114.3273542,"logger":"http","msg":"done waiting on internal rate limiter","identifiers":["test21.whizzygeeks.com"],"ca":"https://acme-v02.api.letsencrypt.org/directory","account":"admin@admin.com"}
{"level":"info","ts":1718622114.3273652,"logger":"http","msg":"using ACME account","account_id":"https://acme-v02.api.letsencrypt.org/acme/acct/1786967397","account_contact":["mailto:admin@admin.com"]}
{"level":"debug","ts":1718622114.327373,"logger":"http.acme_client","msg":"creating order","account":"https://acme-v02.api.letsencrypt.org/acme/acct/1786967397","identifiers":["test21.whizzygeeks.com"]}
{"level":"debug","ts":1718622114.4617774,"logger":"http.acme_client","msg":"http request","method":"HEAD","url":"https://acme-v02.api.letsencrypt.org/acme/new-nonce","headers":{"User-Agent":["Caddy/2.8.0 CertMagic acmez (linux; arm64)"]},"response_headers":{"Cache-Control":["public, max-age=0, no-cache"],"Date":["Mon, 17 Jun 2024 11:01:54 GMT"],"Link":["<https://acme-v02.api.letsencrypt.org/directory>;rel=\"index\""],"Replay-Nonce":["I4RLVp83EI-PUBx-3Uv17NYgQc63TNVB10uO6mT1MjSqgK9h6Uk"],"Server":["nginx"],"Strict-Transport-Security":["max-age=604800"],"X-Frame-Options":["DENY"]},"status_code":200}
{"level":"debug","ts":1718622114.605757,"logger":"http.acme_client","msg":"http request","method":"POST","url":"https://acme-v02.api.letsencrypt.org/acme/new-order","headers":{"Content-Type":["application/jose+json"],"User-Agent":["Caddy/2.8.0 CertMagic acmez (linux; arm64)"]},"response_headers":{"Boulder-Requester":["1786967397"],"Cache-Control":["public, max-age=0, no-cache"],"Content-Length":["328"],"Content-Type":["application/problem+json"],"Date":["Mon, 17 Jun 2024 11:01:54 GMT"],"Link":["<https://acme-v02.api.letsencrypt.org/directory>;rel=\"index\"","<https://letsencrypt.org/docs/rate-limits>;rel=\"help\""],"Replay-Nonce":["prJkcjKSg6yQ7QNvvYwEWec8Bnn2rNfjBpaj6wYSqBPvafWB2EE"],"Retry-After":["81171"],"Server":["nginx"]},"status_code":429}
{"level":"error","ts":1718622114.6058612,"logger":"tls.renew","msg":"could not get certificate from issuer","identifier":"test21.whizzygeeks.com","issuer":"acme-v02.api.letsencrypt.org-directory","error":"HTTP 429 urn:ietf:params:acme:error:rateLimited - Error creating new order :: too many certificates (5) already issued for this exact set of domains in the last 168 hours: test21.whizzygeeks.com, retry after 2024-06-18T09:34:46Z: see https://letsencrypt.org/docs/duplicate-certificate-limit/"}
{"level":"debug","ts":1718622114.6058903,"logger":"events","msg":"event","name":"cert_failed","id":"cdf17c54-f20d-454f-a45d-7c75289f82b2","origin":"tls","data":{"error":{},"identifier":"test21.whizzygeeks.com","issuers":["zerossl","acme-v02.api.letsencrypt.org-directory"],"remaining":7822686034173003,"renewal":true}}
{"level":"error","ts":1718622114.6059027,"logger":"tls.renew","msg":"will retry","error":"[test21.whizzygeeks.com] Renew: [test21.whizzygeeks.com] creating new order: attempt 1: https://acme-v02.api.letsencrypt.org/acme/new-order: HTTP 429 urn:ietf:params:acme:error:rateLimited - Error creating new order :: too many certificates (5) already issued for this exact set of domains in the last 168 hours: test21.whizzygeeks.com, retry after 2024-06-18T09:34:46Z: see https://letsencrypt.org/docs/duplicate-certificate-limit/ (ca=https://acme-v02.api.letsencrypt.org/directory)","attempt":1,"retrying_in":60,"elapsed":0.64077684,"max_duration":2592000}
{"level":"debug","ts":1718622118.4776285,"logger":"http.stdlib","msg":"http: TLS handshake error from 172.30.0.109:1636: EOF"}
{"level":"debug","ts":1718622118.8286824,"logger":"events","msg":"event","name":"tls_get_certificate","id":"76f8a9c7-6e57-40d8-85e4-69221830ffc1","origin":"tls","data":{"client_hello":{"CipherSuites":[4866,4867,4865,49196,49200,159,52393,52392,52394,49195,49199,158,49188,49192,107,49187,49191,103,49162,49172,57,49161,49171,51,157,156,61,60,53,47,255],"ServerName":"test21.whizzygeeks.com","SupportedCurves":[29,23,30,25,24,256,257,258,259,260],"SupportedPoints":"AAEC","SignatureSchemes":[1027,1283,1539,2055,2056,2057,2058,2059,2052,2053,2054,1025,1281,1537,771,769,770,1026,1282,1538],"SupportedProtos":["h2","http/1.1"],"SupportedVersions":[772,771],"RemoteAddr":{"IP":"54.159.182.238","Port":48061,"Zone":""},"LocalAddr":{"IP":"172.30.50.177","Port":443,"Zone":""}}}}
{"level":"debug","ts":1718622118.828721,"logger":"tls.handshake","msg":"choosing certificate","identifier":"test21.whizzygeeks.com","num_choices":2}
{"level":"debug","ts":1718622118.828735,"logger":"tls.handshake","msg":"default certificate selection results","identifier":"test21.whizzygeeks.com","subjects":["test21.whizzygeeks.com"],"managed":true,"issuer_key":"zerossl","hash":"440fa1ce9b97d0d00c406b5aaebbae94d21511935aafd404be9a5af2b5d8af60"}
{"level":"debug","ts":1718622118.8287416,"logger":"tls.handshake","msg":"matched certificate in cache","remote_ip":"54.159.182.238","remote_port":"48061","subjects":["test21.whizzygeeks.com"],"managed":true,"expiration":1726444800,"hash":"440fa1ce9b97d0d00c406b5aaebbae94d21511935aafd404be9a5af2b5d8af60"}
{"level":"warn","ts":1718622118.8287919,"logger":"tls","msg":"certificate is in emergency renewal window; expiration imminent","subjects":["test21.whizzygeeks.com"],"expiration":1726444800,"ari_cert_id":"","next_ari_update":null,"renew_check_interval":4752000,"window_start":-6795364578.8713455,"window_end":-6795364578.8713455,"remaining":7822681.171208327}
{"level":"debug","ts":1718622118.829215,"logger":"tls.on_demand","msg":"certificate expires soon but is already being renewed; serving current certificate","remote_ip":"54.159.182.238","remote_port":"48061","subjects":["test21.whizzygeeks.com"],"remaining":7822681.170787119}
{"level":"debug","ts":1718622118.833376,"logger":"http.handlers.reverse_proxy","msg":"selected upstream","dial":"localhost:6081","total_upstreams":1}
{"level":"debug","ts":1718622118.8337245,"logger":"http.handlers.reverse_proxy","msg":"upstream roundtrip","upstream":"localhost:6081","duration":0.000258532,"request":{"remote_ip":"54.159.182.238","remote_port":"48061","client_ip":"54.159.182.238","proto":"HTTP/2.0","method":"GET","host":"test21.whizzygeeks.com","uri":"/","headers":{"X-Forwarded-Proto":["https"],"Access-Control-Allow-Credentials":["{http.request.access_control_allow_credentials}"],"Access-Control-Max-Age":["{http.request.access_control_max_age}"],"Origin":["{http.request.origin}"],"Access-Control-Allow-Origin":["{http.request.access_control_allow_origin}"],"User-Agent":["curl/7.81.0"],"Accept":["*/*"],"X-Forwarded-For":["54.159.182.238:48061"],"X-Forwarded-Host":["test21.whizzygeeks.com"],"X-Forwarded-Port":[""],"X-Real-Ip":["54.159.182.238:48061"]},"tls":{"resumed":false,"version":772,"cipher_suite":4865,"proto":"h2","server_name":"test21.whizzygeeks.com"}},"headers":{"Host":["test21.whizzygeeks.com"],"X-Varnish":["983045 3"],"Age":["1637"],"Mobile":[""],"Accept-Ranges":["bytes"],"Date":["Mon, 17 Jun 2024 10:34:41 GMT"],"X-Content-Type-Options":["nosniff"],"X-Ratelimit-Remaining":["299"],"Via":["1.1 varnish (Varnish/6.0)"],"X-Cache-Hits":["16"],"Content-Type":["text/html; charset=UTF-8"],"Vary":["Accept-Encoding"],"X-Ratelimit-Limit":["300"],"X-Cache":["HIT"],"Connection":["keep-alive"],"X-Device":["pc"],"Server":["nginx"],"Strict-Transport-Security":["max-age=31536000; includeSubDomains"],"X-Ua":["Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36"]},"status":200}
{"level":"debug","ts":1718622128.4778833,"logger":"http.stdlib","msg":"http: TLS handshake error from 172.30.0.109:36705: EOF"}
{"level":"debug","ts":1718622138.477863,"logger":"http.stdlib","msg":"http: TLS handshake error from 172.30.0.109:35986: EOF"}
{"level":"debug","ts":1718622148.4777973,"logger":"http.stdlib","msg":"http: TLS handshake error from 172.30.0.109:60890: EOF"}
{"level":"debug","ts":1718622158.4778786,"logger":"http.stdlib","msg":"http: TLS handshake error from 172.30.0.109:2442: EOF"}
{"level":"debug","ts":1718622168.4777806,"logger":"http.stdlib","msg":"http: TLS handshake error from 172.30.0.109:30897: EOF"}
{"level":"debug","ts":1718622174.607452,"logger":"tls","msg":"loading managed certificate","domain":"test21.whizzygeeks.com","expiration":1726444800,"issuer_key":"zerossl","storage":"{\"address\":\"primary-redis.whizzygeeks.com:6379\",\"host\":\"primary-redis.whizzygeeks.com\",\"port\":\"6379\",\"db\":6,\"username\":\"\",\"password\":\"REDACTED\",\"timeout\":60,\"key_prefix\":\"caddytls\",\"value_prefix\":\"caddy-storage-redis\",\"aes_key\":\"\",\"tls_enabled\":false,\"tls_insecure\":true}"}
{"level":"info","ts":1718622174.6075747,"logger":"tls.renew","msg":"renewing certificate","identifier":"test21.whizzygeeks.com","remaining":7822625.392429246}
{"level":"debug","ts":1718622174.607619,"logger":"events","msg":"event","name":"cert_obtaining","id":"f19a4fb7-2314-4c58-a5b7-9d48d9d9a544","origin":"tls","data":{"forced":false,"identifier":"test21.whizzygeeks.com","issuer":"zerossl","remaining":7822625392429246,"renewal":true}}
{"level":"info","ts":1718622174.60785,"logger":"tls.issuance.zerossl","msg":"creating certificate","identifiers":["test21.whizzygeeks.com"]}
{"level":"error","ts":1718622174.913276,"logger":"tls.renew","msg":"could not get certificate from issuer","identifier":"test21.whizzygeeks.com","issuer":"zerossl","error":"creating certificate: POST https://api.zerossl.com/certificates?access_key=redacted: HTTP 200: API error 2836: csr_cn_is_invalid (details=map[]) (raw={\"success\":false,\"error\":{\"code\":2836,\"type\":\"csr_cn_is_invalid\"}} decode_error=json: unknown field \"success\")"}
{"level":"info","ts":1718622174.9137535,"logger":"http","msg":"using ACME account","account_id":"https://acme-staging-v02.api.letsencrypt.org/acme/acct/152387183","account_contact":["mailto:admin@admin.com"]}
{"level":"debug","ts":1718622174.9137719,"logger":"http.acme_client","msg":"creating order","account":"https://acme-staging-v02.api.letsencrypt.org/acme/acct/152387183","identifiers":["test21.whizzygeeks.com"]}
{"level":"debug","ts":1718622175.0915434,"logger":"http.acme_client","msg":"http request","method":"HEAD","url":"https://acme-staging-v02.api.letsencrypt.org/acme/new-nonce","headers":{"User-Agent":["Caddy/2.8.0 CertMagic acmez (linux; arm64)"]},"response_headers":{"Cache-Control":["public, max-age=0, no-cache"],"Date":["Mon, 17 Jun 2024 11:02:55 GMT"],"Link":["<https://acme-staging-v02.api.letsencrypt.org/directory>;rel=\"index\""],"Replay-Nonce":["dM8JUU3IqN2oqxlzPBLelXiwqwqFwF19SPqWrMah4dyXnFOGGQY"],"Server":["nginx"],"Strict-Transport-Security":["max-age=604800"],"X-Frame-Options":["DENY"]},"status_code":200}
{"level":"debug","ts":1718622175.1850417,"logger":"http.acme_client","msg":"http request","method":"POST","url":"https://acme-staging-v02.api.letsencrypt.org/acme/new-order","headers":{"Content-Type":["application/jose+json"],"User-Agent":["Caddy/2.8.0 CertMagic acmez (linux; arm64)"]},"response_headers":{"Boulder-Requester":["152387183"],"Cache-Control":["public, max-age=0, no-cache"],"Content-Length":["359"],"Content-Type":["application/json"],"Date":["Mon, 17 Jun 2024 11:02:55 GMT"],"Link":["<https://acme-staging-v02.api.letsencrypt.org/directory>;rel=\"index\""],"Location":["https://acme-staging-v02.api.letsencrypt.org/acme/order/152387183/17256668053"],"Replay-Nonce":["AJh-apMwZ4u3koOC-c3Nn0h59nZuvCpYxkGkDMnOirsPKGOOoZM"],"Server":["nginx"],"Strict-Transport-Security":["max-age=604800"],"X-Frame-Options":["DENY"]},"status_code":201}
{"level":"debug","ts":1718622175.2475388,"logger":"http.acme_client","msg":"http request","method":"POST","url":"https://acme-staging-v02.api.letsencrypt.org/acme/authz-v3/12797115783","headers":{"Content-Type":["application/jose+json"],"User-Agent":["Caddy/2.8.0 CertMagic acmez (linux; arm64)"]},"response_headers":{"Boulder-Requester":["152387183"],"Cache-Control":["public, max-age=0, no-cache"],"Content-Length":["786"],"Content-Type":["application/json"],"Date":["Mon, 17 Jun 2024 11:02:55 GMT"],"Link":["<https://acme-staging-v02.api.letsencrypt.org/directory>;rel=\"index\""],"Replay-Nonce":["dM8JUU3Iu6phD_Wx3-vSb_R-I1uXaT0i-2nZ6HJDKzD_fZsZruI"],"Server":["nginx"],"Strict-Transport-Security":["max-age=604800"],"X-Frame-Options":["DENY"]},"status_code":200}
{"level":"debug","ts":1718622175.2476423,"logger":"http.acme_client","msg":"skipping challenge initiation because authorization is not pending","identifier":"test21.whizzygeeks.com","authz_status":"valid"}
{"level":"info","ts":1718622175.247649,"logger":"http.acme_client","msg":"authorization finalized","identifier":"test21.whizzygeeks.com","authz_status":"valid"}
{"level":"info","ts":1718622175.2476535,"logger":"http.acme_client","msg":"validations succeeded; finalizing order","order":"https://acme-staging-v02.api.letsencrypt.org/acme/order/152387183/17256668053"}
{"level":"debug","ts":1718622175.3330567,"logger":"http.acme_client","msg":"http request","method":"POST","url":"https://acme-staging-v02.api.letsencrypt.org/acme/finalize/152387183/17256668053","headers":{"Content-Type":["application/jose+json"],"User-Agent":["Caddy/2.8.0 CertMagic acmez (linux; arm64)"]},"response_headers":{"Boulder-Requester":["152387183"],"Cache-Control":["public, max-age=0, no-cache"],"Content-Length":["364"],"Content-Type":["application/json"],"Date":["Mon, 17 Jun 2024 11:02:55 GMT"],"Link":["<https://acme-staging-v02.api.letsencrypt.org/directory>;rel=\"index\""],"Location":["https://acme-staging-v02.api.letsencrypt.org/acme/order/152387183/17256668053"],"Replay-Nonce":["dM8JUU3Il_ioed2AbChSX-M-qPhhkJSUm9qsT_khlHHjjnN9hEY"],"Retry-After":["3"],"Server":["nginx"],"Strict-Transport-Security":["max-age=604800"],"X-Frame-Options":["DENY"]},"status_code":200}
{"level":"debug","ts":1718622178.4007277,"logger":"http.acme_client","msg":"http request","method":"POST","url":"https://acme-staging-v02.api.letsencrypt.org/acme/order/152387183/17256668053","headers":{"Content-Type":["application/jose+json"],"User-Agent":["Caddy/2.8.0 CertMagic acmez (linux; arm64)"]},"response_headers":{"Cache-Control":["public, max-age=0, no-cache"],"Content-Length":["471"],"Content-Type":["application/json"],"Date":["Mon, 17 Jun 2024 11:02:58 GMT"],"Link":["<https://acme-staging-v02.api.letsencrypt.org/directory>;rel=\"index\""],"Replay-Nonce":["AJh-apMw5ahTk40O11KVkgH4t7URXrbRCyR-l6F15p2yjMBIH4s"],"Server":["nginx"],"Strict-Transport-Security":["max-age=604800"],"X-Frame-Options":["DENY"]},"status_code":200}
{"level":"debug","ts":1718622178.4658296,"logger":"http.acme_client","msg":"http request","method":"POST","url":"https://acme-staging-v02.api.letsencrypt.org/acme/cert/2b5e737816bc35c063046c7072d011df6a0e","headers":{"Content-Type":["application/jose+json"],"User-Agent":["Caddy/2.8.0 CertMagic acmez (linux; arm64)"]},"response_headers":{"Cache-Control":["public, max-age=0, no-cache"],"Content-Length":["2998"],"Content-Type":["application/pem-certificate-chain"],"Date":["Mon, 17 Jun 2024 11:02:58 GMT"],"Link":["<https://acme-staging-v02.api.letsencrypt.org/directory>;rel=\"index\"","<https://acme-staging-v02.api.letsencrypt.org/acme/cert/2b5e737816bc35c063046c7072d011df6a0e/1>;rel=\"alternate\""],"Replay-Nonce":["AJh-apMw0MF26ppQVwXL2nORnjnE-dea61EDYzBEHbk4aORed5E"],"Server":["nginx"],"Strict-Transport-Security":["max-age=604800"],"X-Frame-Options":["DENY"]},"status_code":200}
{"level":"debug","ts":1718622178.4659343,"logger":"http.acme_client","msg":"getting renewal info","names":["test21.whizzygeeks.com"]}
{"level":"debug","ts":1718622178.4774861,"logger":"http.stdlib","msg":"http: TLS handshake error from 172.30.0.109:59342: EOF"}
{"level":"debug","ts":1718622178.5352857,"logger":"http.acme_client","msg":"http request","method":"GET","url":"https://acme-staging-v02.api.letsencrypt.org/draft-ietf-acme-ari-03/renewalInfo/oXQaBm1Qt4YtSizBfrSNiElszRY.K15zeBa8NcBjBGxwctAR32oO","headers":{"User-Agent":["Caddy/2.8.0 CertMagic acmez (linux; arm64)"]},"response_headers":{"Cache-Control":["public, max-age=0, no-cache"],"Content-Length":["101"],"Content-Type":["application/json"],"Date":["Mon, 17 Jun 2024 11:02:58 GMT"],"Link":["<https://acme-staging-v02.api.letsencrypt.org/directory>;rel=\"index\""],"Retry-After":["21600"],"Server":["nginx"],"Strict-Transport-Security":["max-age=604800"],"X-Frame-Options":["DENY"]},"status_code":200}
{"level":"info","ts":1718622178.535356,"logger":"http.acme_client","msg":"got renewal info","names":["test21.whizzygeeks.com"],"window_start":1723717374,"window_end":1723890174,"selected_time":1723720244,"recheck_after":1718643778.535352,"explanation_url":""}
{"level":"debug","ts":1718622178.603168,"logger":"http.acme_client","msg":"http request","method":"POST","url":"https://acme-staging-v02.api.letsencrypt.org/acme/cert/2b5e737816bc35c063046c7072d011df6a0e/1","headers":{"Content-Type":["application/jose+json"],"User-Agent":["Caddy/2.8.0 CertMagic acmez (linux; arm64)"]},"response_headers":{"Cache-Control":["public, max-age=0, no-cache"],"Content-Length":["2441"],"Content-Type":["application/pem-certificate-chain"],"Date":["Mon, 17 Jun 2024 11:02:58 GMT"],"Link":["<https://acme-staging-v02.api.letsencrypt.org/directory>;rel=\"index\"","<https://acme-staging-v02.api.letsencrypt.org/acme/cert/2b5e737816bc35c063046c7072d011df6a0e/0>;rel=\"alternate\""],"Replay-Nonce":["AJh-apMwicJ0joDc4Oe40dzJPkkZf2x1Iwx3r0Cbr670CMP6AKQ"],"Server":["nginx"],"Strict-Transport-Security":["max-age=604800"],"X-Frame-Options":["DENY"]},"status_code":200}
{"level":"debug","ts":1718622178.6032853,"logger":"http.acme_client","msg":"getting renewal info","names":["test21.whizzygeeks.com"]}
{"level":"debug","ts":1718622178.6664674,"logger":"http.acme_client","msg":"http request","method":"GET","url":"https://acme-staging-v02.api.letsencrypt.org/draft-ietf-acme-ari-03/renewalInfo/oXQaBm1Qt4YtSizBfrSNiElszRY.K15zeBa8NcBjBGxwctAR32oO","headers":{"User-Agent":["Caddy/2.8.0 CertMagic acmez (linux; arm64)"]},"response_headers":{"Cache-Control":["public, max-age=0, no-cache"],"Content-Length":["101"],"Content-Type":["application/json"],"Date":["Mon, 17 Jun 2024 11:02:58 GMT"],"Link":["<https://acme-staging-v02.api.letsencrypt.org/directory>;rel=\"index\""],"Retry-After":["21600"],"Server":["nginx"],"Strict-Transport-Security":["max-age=604800"],"X-Frame-Options":["DENY"]},"status_code":200}
{"level":"info","ts":1718622178.6665208,"logger":"http.acme_client","msg":"got renewal info","names":["test21.whizzygeeks.com"],"window_start":1723717374,"window_end":1723890174,"selected_time":1723724721,"recheck_after":1718643778.6665182,"explanation_url":""}
{"level":"info","ts":1718622178.6665382,"logger":"http.acme_client","msg":"successfully downloaded available certificate chains","count":2,"first_url":"https://acme-staging-v02.api.letsencrypt.org/acme/cert/2b5e737816bc35c063046c7072d011df6a0e"}
{"level":"debug","ts":1718622178.666544,"logger":"http","msg":"selected certificate chain","url":"https://acme-staging-v02.api.letsencrypt.org/acme/cert/2b5e737816bc35c063046c7072d011df6a0e"}
{"level":"info","ts":1718622178.666947,"logger":"http","msg":"waiting on internal rate limiter","identifiers":["test21.whizzygeeks.com"],"ca":"https://acme-v02.api.letsencrypt.org/directory","account":"admin@admin.com"}
{"level":"info","ts":1718622178.6669629,"logger":"http","msg":"done waiting on internal rate limiter","identifiers":["test21.whizzygeeks.com"],"ca":"https://acme-v02.api.letsencrypt.org/directory","account":"admin@admin.com"}
{"level":"info","ts":1718622178.6669734,"logger":"http","msg":"using ACME account","account_id":"https://acme-v02.api.letsencrypt.org/acme/acct/1786967397","account_contact":["mailto:admin@admin.com"]}
{"level":"debug","ts":1718622178.6669805,"logger":"http.acme_client","msg":"creating order","account":"https://acme-v02.api.letsencrypt.org/acme/acct/1786967397","identifiers":["test21.whizzygeeks.com"]}
{"level":"debug","ts":1718622178.7100153,"logger":"http.acme_client","msg":"http request","method":"HEAD","url":"https://acme-v02.api.letsencrypt.org/acme/new-nonce","headers":{"User-Agent":["Caddy/2.8.0 CertMagic acmez (linux; arm64)"]},"response_headers":{"Cache-Control":["public, max-age=0, no-cache"],"Date":["Mon, 17 Jun 2024 11:02:58 GMT"],"Link":["<https://acme-v02.api.letsencrypt.org/directory>;rel=\"index\""],"Replay-Nonce":["prJkcjKSJBwQC_O7Wb8UOsKp5d0A4M3lf1SyJJzxh3nrS5d5GUw"],"Server":["nginx"],"Strict-Transport-Security":["max-age=604800"],"X-Frame-Options":["DENY"]},"status_code":200}
{"level":"debug","ts":1718622178.7661018,"logger":"http.acme_client","msg":"http request","method":"POST","url":"https://acme-v02.api.letsencrypt.org/acme/new-order","headers":{"Content-Type":["application/jose+json"],"User-Agent":["Caddy/2.8.0 CertMagic acmez (linux; arm64)"]},"response_headers":{"Boulder-Requester":["1786967397"],"Cache-Control":["public, max-age=0, no-cache"],"Content-Length":["328"],"Content-Type":["application/problem+json"],"Date":["Mon, 17 Jun 2024 11:02:58 GMT"],"Link":["<https://acme-v02.api.letsencrypt.org/directory>;rel=\"index\"","<https://letsencrypt.org/docs/rate-limits>;rel=\"help\""],"Replay-Nonce":["prJkcjKSzubxFw-nx7gbKdxPm0PnV1DvBiQnDNk0PGytwbK55gM"],"Retry-After":["81107"],"Server":["nginx"]},"status_code":429}
{"level":"error","ts":1718622178.7662132,"logger":"tls.renew","msg":"could not get certificate from issuer","identifier":"test21.whizzygeeks.com","issuer":"acme-v02.api.letsencrypt.org-directory","error":"HTTP 429 urn:ietf:params:acme:error:rateLimited - Error creating new order :: too many certificates (5) already issued for this exact set of domains in the last 168 hours: test21.whizzygeeks.com, retry after 2024-06-18T09:34:46Z: see https://letsencrypt.org/docs/duplicate-certificate-limit/"}
{"level":"debug","ts":1718622178.7662432,"logger":"events","msg":"event","name":"cert_failed","id":"836bec2a-9491-4b03-a6c7-be5583ecb96d","origin":"tls","data":{"error":{},"identifier":"test21.whizzygeeks.com","issuers":["zerossl","acme-v02.api.letsencrypt.org-directory"],"remaining":7822625392429246,"renewal":true}}
{"level":"error","ts":1718622178.766253,"logger":"tls.renew","msg":"will retry","error":"[test21.whizzygeeks.com] Renew: [test21.whizzygeeks.com] creating new order: attempt 1: https://acme-v02.api.letsencrypt.org/acme/new-order: HTTP 429 urn:ietf:params:acme:error:rateLimited - Error creating new order :: too many certificates (5) already issued for this exact set of domains in the last 168 hours: test21.whizzygeeks.com, retry after 2024-06-18T09:34:46Z: see https://letsencrypt.org/docs/duplicate-certificate-limit/ (ca=https://acme-v02.api.letsencrypt.org/directory)","attempt":2,"retrying_in":120,"elapsed":64.801127187,"max_duration":2592000}
{"level":"debug","ts":1718622188.4774888,"logger":"http.stdlib","msg":"http: TLS handshake error from 172.30.0.109:18773: EOF"}
{"level":"debug","ts":1718622198.477621,"logger":"http.stdlib","msg":"http: TLS handshake error from 172.30.0.109:53509: EOF"}
{"level":"debug","ts":1718622208.4777474,"logger":"http.stdlib","msg":"http: TLS handshake error from 172.30.0.109:7492: EOF"}
{"level":"debug","ts":1718622218.477652,"logger":"http.stdlib","msg":"http: TLS handshake error from 172.30.0.109:61529: EOF"}
{"level":"debug","ts":1718622228.4774976,"logger":"http.stdlib","msg":"http: TLS handshake error from 172.30.0.109:33741: EOF"}
{"level":"debug","ts":1718622238.4773412,"logger":"http.stdlib","msg":"http: TLS handshake error from 172.30.0.109:44118: EOF"}
{"level":"debug","ts":1718622240.5106258,"logger":"http.stdlib","msg":"http: TLS handshake error from 99.82.161.83:33280: EOF"}
{"level":"debug","ts":1718622248.4775677,"logger":"http.stdlib","msg":"http: TLS handshake error from 172.30.0.109:59503: EOF"}
{"level":"debug","ts":1718622258.4776495,"logger":"http.stdlib","msg":"http: TLS handshake error from 172.30.0.109:56593: EOF"}
{"level":"debug","ts":1718622268.4778032,"logger":"http.stdlib","msg":"http: TLS handshake error from 172.30.0.109:61862: EOF"}
3. Caddy version:
Caddy Version 2.8.0
4. How I installed and ran Caddy:
a. System environment:
Ubuntu 22.04.4 LTS on ARM AWS EC2 instance
b. Command:
systemctl start caddy
c. Service/unit/compose file:
[Unit]
Description=Caddy
Documentation=https://caddyserver.com/docs/
After=network.target network-online.target
Requires=network-online.target
[Service]
Type=notify
User=caddy
Group=caddy
ExecStart=/usr/bin/caddy run --environ --config /etc/caddy/Caddyfile
ExecReload=/usr/bin/caddy reload --config /etc/caddy/Caddyfile --force
TimeoutStopSec=5s
LimitNOFILE=1048576
PrivateTmp=true
ProtectSystem=full
AmbientCapabilities=CAP_NET_ADMIN CAP_NET_BIND_SERVICE
[Install]
WantedBy=multi-user.target
d. My complete Caddy config:
{
debug
order rate_limit before basicauth
#cert_issuer acme https://acme-v02.api.letsencrypt.org/directory
#
#cert_issuer acme https://acme-v02.api.letsencrypt.org/directory
cert_issuer zerossl XXXXXXXXXXXXXXXXXXXXXXXX
cert_issuer acme https://acme-v02.api.letsencrypt.org/directory
# Use Let's Encrypt or ZeroSSL to sign certificates.
log {
output file /var/log/caddy/server.log {
roll_size 100mb
roll_keep 15
roll_keep_for 480h
}
}
email admin@whizzygeeks.com
storage redis {
host "primary-redis.whizzygeeksview.com"
port 6379
address "primary-redis.whizzygeeksview.com:6379"
username ""
password "Klinsmen80"
db 6
key_prefix "caddytls"
value_prefix "caddy-storage-redis"
timeout 60
tls_enabled "false"
tls_insecure "true"
}
on_demand_tls {
ask http://domain-validator.whizzygeeks.com/px-validate
}
renew_interval 55d
#ocsp_interval 1d
}
*.whizzygeeks.com:80 {
header / {
-Server
}
@post method POST PUT OPTIONS
@get method GET
reverse_proxy @post internal-PROD-whizzygeeks-THEME-INTERNAL-ALB-XXXXXXXX.us-east-1.elb.amazonaws.com:80 {
method POST
header_up Host {http.request.host}
header_up X-Real-IP {http.request.remote}
header_up X-Forwarded-For {http.request.remote}
header_up X-Forwarded-Port {http.request.port}
#header_up CloudFront-Is-Mobile-Viewer {http.request.header.cloudfront_is_mobile_viewer}
#header_up CloudFront-Is-IOS-Viewer {http.request.header.cloudfront_is_ios_viewer}
#header_up CloudFront-Is-Android-Viewer {http.request.header.cloudfront_is_android_viewer}
#header_up CloudFront-Is-Desktop-Viewer {http.request.header.cloudfront_is_desktop_viewer}
header_up Access-Control-Allow-Origin {http.request.access_control_allow_origin}
header_up Access-Control-Allow-Credentials {http.request.access_control_allow_credentials}
header_up Access-Control-Max-Age {http.request.access_control_max_age}
header_up Origin {http.request.origin}
}
reverse_proxy @post internal-PROD-whizzygeeks-THEME-INTERNAL-ALB-XXXXXXXXXXXX.us-east-1.elb.amazonaws.com:80 {
method OPTIONS
header_up Host {http.request.host}
header_up X-Real-IP {http.request.remote}
header_up X-Forwarded-For {http.request.remote}
header_up X-Forwarded-Port {http.request.port}
#header_up CloudFront-Is-Mobile-Viewer {http.request.cloudfront_is_mobile_viewer}
#header_up CloudFront-Is-IOS-Viewer {http.request.cloudfront_is_ios_viewer}
#header_up CloudFront-Is-Android-Viewer {http.request.cloudfront_is_android_viewer}
#header_up CloudFront-Is-Desktop-Viewer {http.request.cloudfront_is_desktop_viewer}
header_up Access-Control-Allow-Origin {http.request.access_control_allow_origin}
header_up Access-Control-Allow-Credentials {http.request.access_control_allow_credentials}
header_up Access-Control-Max-Age {http.request.access_control_max_age}
header_up Origin {http.request.origin}
}
reverse_proxy @post internal-PROD-whizzygeeks-THEME-INTERNAL-ALB-XXXXXXXXXXXX.us-east-1.elb.amazonaws.com:80 {
method PUT
header_up Host {http.request.host}
header_up X-Real-IP {http.request.remote}
header_up X-Forwarded-For {http.request.remote}
header_up X-Forwarded-Port {http.request.port}
#header_up CloudFront-Is-Mobile-Viewer {http.request.cloudfront_is_mobile_viewer}
#header_up CloudFront-Is-IOS-Viewer {http.request.cloudfront_is_ios_viewer}
#header_up CloudFront-Is-Android-Viewer {http.request.cloudfront_is_android_viewer}
#header_up CloudFront-Is-Desktop-Viewer {http.request.cloudfront_is_desktop_viewer}
header_up Access-Control-Allow-Origin {http.request.access_control_allow_origin}
header_up Access-Control-Allow-Credentials {http.request.access_control_allow_credentials}
header_up Access-Control-Max-Age {http.request.access_control_max_age}
header_up Origin {http.request.origin}
}
reverse_proxy @get http://localhost:6081 {
method GET
header_up Host {http.request.host}
header_up X-Real-IP {http.request.remote}
header_up X-Forwarded-For {http.request.remote}
header_up X-Forwarded-Port {http.request.port}
#header_up CloudFront-Is-Mobile-Viewer {http.request.header.cloudfront_is_mobile_viewer}
#header_up CloudFront-Is-IOS-Viewer {http.request.header.cloudfront_is_ios_viewer}
#header_up CloudFront-Is-Android-Viewer {http.request.header.cloudfront_is_android_viewer}
#header_up CloudFront-Is-Desktop-Viewer {http.request.header.cloudfront_is_desktop_viewer}
header_up Access-Control-Allow-Origin {http.request.access_control_allow_origin}
header_up Access-Control-Allow-Credentials {http.request.access_control_allow_credentials}
header_up Access-Control-Max-Age {http.request.access_control_max_age}
header_up Origin {http.request.origin}
}
}
http:// {
redir https://{host}{uri} permanent
}
:443 {
header / {
-Server
}
log {
output file /var/log/caddy/wildcard_access.log {
roll_size 100mb
roll_keep 20
roll_keep_for 480h
}
}
#tls zerossl https://acme-v02.api.letsencrypt.org/directory
tls {
#issuer zerossl # could alternatively use "issuer acme ..." with ZeroSSL directory endpoint
#issuer acme # current default is LE; hard-code dir endpoint to force that
on_demand
#issuer zerossl acme
}
@post method POST PUT OPTIONS
@get method GET
reverse_proxy @post internal-PROD-whizzygeeks-THEME-INTERNAL-ALB-XXXXXXXXXXXX.us-east-1.elb.amazonaws.com:80 {
method POST
header_up Host {http.request.host}
header_up X-Real-IP {http.request.remote}
header_up X-Forwarded-For {http.request.remote}
header_up X-Forwarded-Port {http.request.port}
header_up Access-Control-Allow-Origin {http.request.access_control_allow_origin}
header_up Access-Control-Allow-Credentials {http.request.access_control_allow_credentials}
header_up Access-Control-Max-Age {http.request.access_control_max_age}
header_up Origin {http.request.origin}
}
reverse_proxy @post internal-PROD-whizzygeeks-THEME-INTERNAL-ALB-XXXXXXXXXXXX.us-east-1.elb.amazonaws.com:80 {
method OPTIONS
header_up Host {http.request.host}
header_up X-Real-IP {http.request.remote}
header_up X-Forwarded-For {http.request.remote}
header_up X-Forwarded-Port {http.request.port}
header_up Access-Control-Allow-Origin {http.request.access_control_allow_origin}
header_up Access-Control-Allow-Credentials {http.request.access_control_allow_credentials}
header_up Access-Control-Max-Age {http.request.access_control_max_age}
header_up Origin {http.request.origin}
}
reverse_proxy @post internal-PROD-whizzygeeks-THEME-INTERNAL-ALB-XXXXXXXXXXXX.us-east-1.elb.amazonaws.com:80 {
method PUT
header_up Host {http.request.host}
header_up X-Real-IP {http.request.remote}
header_up X-Forwarded-For {http.request.remote}
header_up X-Forwarded-Port {http.request.port}
header_up Access-Control-Allow-Origin {http.request.access_control_allow_origin}
header_up Access-Control-Allow-Credentials {http.request.access_control_allow_credentials}
header_up Access-Control-Max-Age {http.request.access_control_max_age}
header_up Origin {http.request.origin}
}
reverse_proxy @get http://localhost:6081 {
method GET
header_up Host {http.request.host}
header_up X-Real-IP {http.request.remote}
header_up X-Forwarded-For {http.request.remote}
header_up X-Forwarded-Port {http.request.port}
header_up Access-Control-Allow-Origin {http.request.access_control_allow_origin}
header_up Access-Control-Allow-Credentials {http.request.access_control_allow_credentials}
header_up Access-Control-Max-Age {http.request.access_control_max_age}
header_up Origin {http.request.origin}
}
rate_limit {
zone dynamic_example {
key {remote_host}
events 600
window 1m
jitter 2.0
sweep_interval 1m
}
}
}