1. The problem I’m having:
I am trying to setup on-demand tls. Things are working fine when I am using acme module i.e. Letsencrypt as a CA. However, when using Zerossl as an issuer, certificates are not issued and the request is stuck.
2. Error messages and/or full log output:
{"level":"debug","ts":1680766494.4370844,"logger":"events","msg":"event","name":"tls_get_certificate","id":"38439e30-5686-46a1-8c49-8a5f468a209e","origin":"tls","data":{"client_hello":{"CipherSuites":[31354,4865,4866,4867,49195,49199,49196,49200,52393,52392,49171,49172,156,157,47,53],"ServerName":"v1.dvs.beta.postman.wtf","SupportedCurves":[10794,29,23,24],"SupportedPoints":"AA==","SignatureSchemes":[1027,2052,1025,1283,2053,1281,2054,1537],"SupportedProtos":["h2","http/1.1"],"SupportedVersions":[27242,772,771],"Conn":{}}}}
{"level":"debug","ts":1680766494.4371305,"logger":"tls.handshake","msg":"no matching certificates and no custom selection logic","identifier":"v1.dvs.beta.postman.wtf"}
{"level":"debug","ts":1680766494.437137,"logger":"tls.handshake","msg":"no matching certificates and no custom selection logic","identifier":"*.dvs.beta.postman.wtf"}
{"level":"debug","ts":1680766494.4371402,"logger":"tls.handshake","msg":"no matching certificates and no custom selection logic","identifier":"*.*.beta.postman.wtf"}
{"level":"debug","ts":1680766494.4371433,"logger":"tls.handshake","msg":"no matching certificates and no custom selection logic","identifier":"*.*.*.postman.wtf"}
{"level":"debug","ts":1680766494.4371464,"logger":"tls.handshake","msg":"no matching certificates and no custom selection logic","identifier":"*.*.*.*.wtf"}
{"level":"debug","ts":1680766494.4371493,"logger":"tls.handshake","msg":"no matching certificates and no custom selection logic","identifier":"*.*.*.*.*"}
{"level":"debug","ts":1680766494.4371533,"logger":"tls.handshake","msg":"all external certificate managers yielded no certificates and no errors","remote_ip":"10.100.27.62","remote_port":"62946","sni":"v1.dvs.beta.postman.wtf"}
{"level":"debug","ts":1680766494.487104,"logger":"tls","msg":"response from ask endpoint","domain":"v1.dvs.beta.postman.wtf","url":"https://hosted-service.postman-beta.tech/domains/isVerified?domain=v1.dvs.beta.postman.wtf","status":200}
{"level":"info","ts":1680766494.4871306,"logger":"tls.on_demand","msg":"obtaining new certificate","remote_ip":"10.100.27.62","remote_port":"62946","server_name":"v1.dvs.beta.postman.wtf"}
{"level":"info","ts":1680766494.487397,"logger":"tls.obtain","msg":"acquiring lock","identifier":"v1.dvs.beta.postman.wtf"}
{"level":"info","ts":1680766494.4886575,"logger":"tls.obtain","msg":"lock acquired","identifier":"v1.dvs.beta.postman.wtf"}
{"level":"info","ts":1680766494.4887426,"logger":"tls.obtain","msg":"obtaining certificate","identifier":"v1.dvs.beta.postman.wtf"}
{"level":"debug","ts":1680766494.4888098,"logger":"events","msg":"event","name":"cert_obtaining","id":"cb42f77c-e94d-4488-b964-6715d01b21b9","origin":"tls","data":{"identifier":"v1.dvs.beta.postman.wtf"}}
{"level":"debug","ts":1680766494.4890897,"logger":"tls.obtain","msg":"trying issuer 1/1","issuer":"acme.zerossl.com-v2-DV90"}
{"level":"info","ts":1680766494.489294,"logger":"http","msg":"waiting on internal rate limiter","identifiers":["v1.dvs.beta.postman.wtf"],"ca":"https://acme.zerossl.com/v2/DV90","account":"maaiz.elahi@postman.com"}
{"level":"info","ts":1680766494.4893131,"logger":"http","msg":"done waiting on internal rate limiter","identifiers":["v1.dvs.beta.postman.wtf"],"ca":"https://acme.zerossl.com/v2/DV90","account":"maaiz.elahi@postman.com"}
{"level":"debug","ts":1680766494.975301,"logger":"http.acme_client","msg":"http request","method":"HEAD","url":"https://acme.zerossl.com/v2/DV90/newNonce","headers":{"User-Agent":["Caddy/2.6.4 CertMagic acmez (linux; amd64)"]},"response_headers":{"Access-Control-Allow-Origin":["*"],"Cache-Control":["max-age=0, no-cache, no-store"],"Content-Type":["application/octet-stream"],"Date":["Thu, 06 Apr 2023 07:34:54 GMT"],"Link":["<https://acme.zerossl.com/v2/DV90>;rel=\"index\""],"Replay-Nonce":["mUoI9QYVQnyuZK6BMZaBYOjNgBnXXE4_6zvmdytZmpw"],"Server":["nginx"],"Strict-Transport-Security":["max-age=15724800; includeSubDomains"]},"status_code":200}
{"level":"debug","ts":1680766495.4514143,"logger":"http.acme_client","msg":"http request","method":"POST","url":"https://acme.zerossl.com/v2/DV90/newOrder","headers":{"Content-Type":["application/jose+json"],"User-Agent":["Caddy/2.6.4 CertMagic acmez (linux; amd64)"]},"response_headers":{"Access-Control-Allow-Origin":["*"],"Cache-Control":["max-age=0, no-cache, no-store"],"Content-Length":["285"],"Content-Type":["application/json"],"Date":["Thu, 06 Apr 2023 07:34:55 GMT"],"Location":["https://acme.zerossl.com/v2/DV90/order/Z0YE5I6hI7g8wBGEf05E_A"],"Replay-Nonce":["JDX7sBnI3eekPd4xUeQUf8nh2NcjJs6pW_6TQDCGDiM"],"Server":["nginx"],"Strict-Transport-Security":["max-age=15724800; includeSubDomains"]},"status_code":201}
{"level":"debug","ts":1680766495.8331258,"logger":"http.acme_client","msg":"http request","method":"POST","url":"https://acme.zerossl.com/v2/DV90/authz/rmT3oS2zvsH1mqiVuHT99Q","headers":{"Content-Type":["application/jose+json"],"User-Agent":["Caddy/2.6.4 CertMagic acmez (linux; amd64)"]},"response_headers":{"Access-Control-Allow-Origin":["*"],"Cache-Control":["max-age=0, no-cache, no-store"],"Content-Length":["453"],"Content-Type":["application/json"],"Date":["Thu, 06 Apr 2023 07:34:55 GMT"],"Link":["<https://acme.zerossl.com/v2/DV90>;rel=\"index\""],"Replay-Nonce":["gulHceVQIkBsO20qHPHCb-MxFrCLjpk4OCan7_CP4wY"],"Retry-After":["5"],"Server":["nginx"],"Strict-Transport-Security":["max-age=15724800; includeSubDomains"]},"status_code":200}
{"level":"info","ts":1680766495.8332279,"logger":"http.acme_client","msg":"trying to solve challenge","identifier":"v1.dvs.beta.postman.wtf","challenge_type":"http-01","ca":"https://acme.zerossl.com/v2/DV90"}
{"level":"debug","ts":1680766495.833524,"logger":"http.acme_client","msg":"waiting for solver before continuing","identifier":"v1.dvs.beta.postman.wtf","challenge_type":"http-01"}
{"level":"debug","ts":1680766495.8335366,"logger":"http.acme_client","msg":"done waiting for solver","identifier":"v1.dvs.beta.postman.wtf","challenge_type":"http-01"}
{"level":"debug","ts":1680766496.253525,"logger":"http.acme_client","msg":"http request","method":"POST","url":"https://acme.zerossl.com/v2/DV90/chall/oEFnPP-6LYWqz_hyRkU_vg","headers":{"Content-Type":["application/jose+json"],"User-Agent":["Caddy/2.6.4 CertMagic acmez (linux; amd64)"]},"response_headers":{"Access-Control-Allow-Origin":["*"],"Cache-Control":["max-age=0, no-cache, no-store"],"Content-Length":["164"],"Content-Type":["application/json"],"Date":["Thu, 06 Apr 2023 07:34:56 GMT"],"Link":["<https://acme.zerossl.com/v2/DV90/authz/rmT3oS2zvsH1mqiVuHT99Q>;rel=\"up\""],"Replay-Nonce":["6z0cKjO5Jeqb-KISnis1dGn0tZu02eTNDAS2rYEwLmA"],"Retry-After":["10"],"Server":["nginx"],"Strict-Transport-Security":["max-age=15724800; includeSubDomains"]},"status_code":200}
{"level":"debug","ts":1680766496.2535858,"logger":"http.acme_client","msg":"challenge accepted","identifier":"v1.dvs.beta.postman.wtf","challenge_type":"http-01"}
{"level":"info","ts":1680766496.839228,"logger":"http","msg":"served key authentication","identifier":"v1.dvs.beta.postman.wtf","challenge":"http-01","remote":"10.100.27.62:26008","distributed":false}
{"level":"debug","ts":1680766497.158026,"logger":"http.acme_client","msg":"http request","method":"POST","url":"https://acme.zerossl.com/v2/DV90/authz/rmT3oS2zvsH1mqiVuHT99Q","headers":{"Content-Type":["application/jose+json"],"User-Agent":["Caddy/2.6.4 CertMagic acmez (linux; amd64)"]},"response_headers":{"Access-Control-Allow-Origin":["*"],"Cache-Control":["max-age=0, no-cache, no-store"],"Content-Length":["323"],"Content-Type":["application/json"],"Date":["Thu, 06 Apr 2023 07:34:57 GMT"],"Link":["<https://acme.zerossl.com/v2/DV90>;rel=\"index\""],"Replay-Nonce":["frcMrMUxDQz8A0xTz6Ucf2XrsdhO_pGvWf1dnHl7rmo"],"Retry-After":["300"],"Server":["nginx"],"Strict-Transport-Security":["max-age=15724800; includeSubDomains"]},"status_code":200}
{"level":"info","ts":1680766497.1582177,"logger":"http.acme_client","msg":"authorization finalized","identifier":"v1.dvs.beta.postman.wtf","authz_status":"valid"}
{"level":"info","ts":1680766497.158233,"logger":"http.acme_client","msg":"validations succeeded; finalizing order","order":"https://acme.zerossl.com/v2/DV90/order/Z0YE5I6hI7g8wBGEf05E_A"}
{"level":"debug","ts":1680766497.6260126,"logger":"http.acme_client","msg":"http request","method":"POST","url":"https://acme.zerossl.com/v2/DV90/order/Z0YE5I6hI7g8wBGEf05E_A/finalize","headers":{"Content-Type":["application/jose+json"],"User-Agent":["Caddy/2.6.4 CertMagic acmez (linux; amd64)"]},"response_headers":{"Access-Control-Allow-Origin":["*"],"Cache-Control":["max-age=0, no-cache, no-store"],"Content-Length":["288"],"Content-Type":["application/json"],"Date":["Thu, 06 Apr 2023 07:34:57 GMT"],"Location":["https://acme.zerossl.com/v2/DV90/order/Z0YE5I6hI7g8wBGEf05E_A"],"Replay-Nonce":["ZBH0eucGm5Qxu68dAkG_41cESOjZAtx5bBl16297w_s"],"Retry-After":["15"],"Server":["nginx"],"Strict-Transport-Security":["max-age=15724800; includeSubDomains"]},"status_code":200}
{"level":"debug","ts":1680766502.2318375,"logger":"http.acme_client","msg":"http request","method":"POST","url":"https://acme.zerossl.com/v2/DV90/order/Q_FkbI5BqRWdPwTRkJelgg","headers":{"Content-Type":["application/jose+json"],"User-Agent":["Caddy/2.6.4 CertMagic acmez (linux; amd64)"]},"response_headers":{"Access-Control-Allow-Origin":["*"],"Cache-Control":["max-age=0, no-cache, no-store"],"Content-Length":["288"],"Content-Type":["application/json"],"Date":["Thu, 06 Apr 2023 07:35:02 GMT"],"Location":["https://acme.zerossl.com/v2/DV90/order/Q_FkbI5BqRWdPwTRkJelgg"],"Replay-Nonce":["m3k4gmyoiH0Z_ZVcmn_gBWU3BMZeaRuH2uMAJY6-PFU"],"Retry-After":["15"],"Server":["nginx"],"Strict-Transport-Security":["max-age=15724800; includeSubDomains"]},"status_code":200}
3. Caddy version:
caddy-2.6.4
4. How I installed and ran Caddy:
a. System environment:
Caddy alpine docker image
b. Command:
command: ["caddy", "run", "--config", "/config/caddy/caddy.json"]
c. Service/unit/compose file:
Used kubernetes deployment
d. My complete Caddy config:
{
"apps": {
"http": {
"http_port": 80,
"servers": {
"https": {
"listen": [":443"],
"routes": [
{
"handle": [
{
"handler": "reverse_proxy",
"transport": { "protocol": "http", "tls": {} },
"upstreams": [{ "dial": "localhost:9999" }]
}
],
"terminal": true
}
],
"tls_connection_policies": [{}]
},
"health-check": {
"listen": [":9999"],
"routes": [{ "handle": [{ "body": "OK", "handler": "static_response" }], "match": [{ "path": ["/"] }] }]
}
}
},
"tls": {
"automation": {
"on_demand": { "ask": "http://localhost:9999" },
"policies": [
{
"issuers": [{ "module": "zerossl", "email": "maaiz.elahi@postman.com", "api_key": "374994a40d168a635267180b15d5bffc" }],
"on_demand": true
}
]
}
}
},
"logging": { "logs": { "default": { "level": "DEBUG", "writer": { "output": "stdout" } } } }
}