Let'sEncrypt ACME DNS challenge certificate not trusted

1. The problem I’m having:

I am getting errors that my certificates are not trusted when using Let’sEncrypt and ACME DNS challenge using Porkbun API. The certificate is successfully acquired (though it takes a few attempts normally). I’m using a local-only reverse proxy to a domain I recently purchased. When I hit the “service.quesadilla.cloud” with Firefox of Safari, I have to agree to visit an “unsafe” site. Firefox tells me “Error code: SEC_ERROR_UNKNOWN_ISSUER”.

FYI: “docker.richard” is a local DNS record that points to the IP of my docker machine.

2. Error messages and/or full log output:

These logs start when I ran:

docker restart caddy

and end after I agreed to open the “unsafe” website on Firefox inside my local network.

{"level":"info","ts":1706209045.1820817,"msg":"using provided configuration","config_file":"/etc/caddy/Caddyfile","config_adapter":"caddyfile"}

{"level":"info","ts":1706209045.1919286,"logger":"admin","msg":"admin endpoint started","address":"localhost:2019","enforce_origin":false,"origins":["//localhost:2019","//[::1]:2019","//127.0.0.1:2019"]}

{"level":"info","ts":1706209045.1927032,"logger":"http.auto_https","msg":"server is listening only on the HTTPS port but has no TLS connection policies; adding one to enable TLS","server_name":"srv0","https_port":443}

{"level":"info","ts":1706209045.1927836,"logger":"http.auto_https","msg":"enabling automatic HTTP->HTTPS redirects","server_name":"srv0"}

{"level":"debug","ts":1706209045.1928606,"logger":"http.auto_https","msg":"adjusted config","tls":{"automation":{"policies":[{"subjects":["dozzle.quesadilla.cloud"]},{}]}},"http":{"servers":{"remaining_auto_https_redirects":{"listen":[":80"],"routes":[{},{}]},"srv0":{"listen":[":443"],"routes":[{"handle":[{"handler":"subroute","routes":[{"handle":[{"handler":"reverse_proxy","upstreams":[{"dial":"docker.richard:9999"}]}]}]}],"terminal":true}],"tls_connection_policies":[{}],"automatic_https":{}}}}}

{"level":"info","ts":1706209045.1934645,"logger":"tls.cache.maintenance","msg":"started background certificate maintenance","cache":"0xc0002f6200"}

{"level":"info","ts":1706209045.1941743,"logger":"http","msg":"enabling HTTP/3 listener","addr":":443"}

{"level":"debug","ts":1706209045.1947746,"logger":"http","msg":"starting server loop","address":"[::]:443","tls":true,"http3":true}

{"level":"info","ts":1706209045.1948202,"logger":"http.log","msg":"server running","name":"srv0","protocols":["h1","h2","h3"]}

{"level":"debug","ts":1706209045.195001,"logger":"http","msg":"starting server loop","address":"[::]:80","tls":false,"http3":false}

{"level":"info","ts":1706209045.1950786,"logger":"http.log","msg":"server running","name":"remaining_auto_https_redirects","protocols":["h1","h2","h3"]}

{"level":"info","ts":1706209045.1950998,"logger":"http","msg":"enabling automatic TLS certificate management","domains":["dozzle.quesadilla.cloud"]}

{"level":"debug","ts":1706209045.1970606,"logger":"tls.cache","msg":"added certificate to cache","subjects":["dozzle.quesadilla.cloud"],"expiration":1713728084,"managed":true,"issuer_key":"acme-staging-v02.api.letsencrypt.org-directory","hash":"7b0fe6a3f09d0f023a9502ed0dcf4176590c4194d7b0e5b9837c3cf7d6c74cf6","cache_size":1,"cache_capacity":10000}

{"level":"debug","ts":1706209045.197129,"logger":"events","msg":"event","name":"cached_managed_cert","id":"8022e3b3-cf6f-4c9c-9652-d84f334189ae","origin":"tls","data":{"sans":["dozzle.quesadilla.cloud"]}}

{"level":"info","ts":1706209045.1978443,"msg":"autosaved config (load with --resume flag)","file":"/config/caddy/autosave.json"}

{"level":"info","ts":1706209045.1980448,"msg":"serving initial configuration"}

{"level":"warn","ts":1706209045.1993315,"logger":"tls","msg":"storage cleaning happened too recently; skipping for now","storage":"FileStorage:/data/caddy","instance":"0acd6c77-e770-4a12-8afc-c2227ab75058","try_again":1706295445.1993277,"try_again_in":86399.999999105}

{"level":"info","ts":1706209045.1996849,"logger":"tls","msg":"finished cleaning storage units"}

{"level":"debug","ts":1706209049.0600615,"logger":"events","msg":"event","name":"tls_get_certificate","id":"805319b0-d5cb-4f0b-8c55-5c6eeecfdb3c","origin":"tls","data":{"client_hello":{"CipherSuites":[4865,4867,4866,49195,49199,52393,52392,49196,49200,49162,49161,49171,49172,156,157,47,53],"ServerName":"dozzle.quesadilla.cloud","SupportedCurves":[29,23,24,25,256,257],"SupportedPoints":"AA==","SignatureSchemes":[1027,1283,1539,2052,2053,2054,1025,1281,1537],"SupportedProtos":["h2","http/1.1"],"SupportedVersions":[772,771],"RemoteAddr":{"IP":"192.168.35.12","Port":54610,"Zone":""},"LocalAddr":{"IP":"172.20.0.8","Port":443,"Zone":""}}}}

{"level":"debug","ts":1706209049.0606053,"logger":"tls.handshake","msg":"choosing certificate","identifier":"dozzle.quesadilla.cloud","num_choices":1}

{"level":"debug","ts":1706209049.0609682,"logger":"tls.handshake","msg":"default certificate selection results","identifier":"dozzle.quesadilla.cloud","subjects":["dozzle.quesadilla.cloud"],"managed":true,"issuer_key":"acme-staging-v02.api.letsencrypt.org-directory","hash":"7b0fe6a3f09d0f023a9502ed0dcf4176590c4194d7b0e5b9837c3cf7d6c74cf6"}

{"level":"debug","ts":1706209049.0610912,"logger":"tls.handshake","msg":"matched certificate in cache","remote_ip":"192.168.35.12","remote_port":"54610","subjects":["dozzle.quesadilla.cloud"],"managed":true,"expiration":1713728084,"hash":"7b0fe6a3f09d0f023a9502ed0dcf4176590c4194d7b0e5b9837c3cf7d6c74cf6"}

{"level":"debug","ts":1706209049.090824,"logger":"http.handlers.reverse_proxy","msg":"selected upstream","dial":"docker.richard:9999","total_upstreams":1}

{"level":"debug","ts":1706209049.1171517,"logger":"http.handlers.reverse_proxy","msg":"upstream roundtrip","upstream":"docker.richard:9999","duration":0.026152441,"request":{"remote_ip":"192.168.35.12","remote_port":"54610","client_ip":"192.168.35.12","proto":"HTTP/2.0","method":"GET","host":"dozzle.quesadilla.cloud","uri":"/api/events/stream","headers":{"Te":["trailers"],"X-Forwarded-Proto":["https"],"Referer":["https://dozzle.quesadilla.cloud/"],"Sec-Fetch-Site":["same-origin"],"Accept-Language":["en-US,en;q=0.5"],"Sec-Fetch-Mode":["cors"],"User-Agent":["Mozilla/5.0 (X11; Linux x86_64; rv:121.0) Gecko/20100101 Firefox/121.0"],"X-Forwarded-For":["192.168.35.12"],"Pragma":["no-cache"],"Accept-Encoding":["gzip, deflate, br"],"Sec-Fetch-Dest":["empty"],"Cache-Control":["no-cache"],"Accept":["text/event-stream"],"X-Forwarded-Host":["dozzle.quesadilla.cloud"]},"tls":{"resumed":false,"version":772,"cipher_suite":4867,"proto":"h2","server_name":"dozzle.quesadilla.cloud"}},"headers":{"Cache-Control":["no-transform","no-cache"],"Connection":["keep-alive"],"Content-Security-Policy":["default-src 'none'; script-src 'self'; style-src 'self' 'unsafe-inline'; img-src 'self' gravatar.com data:; manifest-src 'self'; connect-src 'self' api.github.com;"],"Content-Type":["text/event-stream"],"X-Accel-Buffering":["no"],"Date":["Thu, 25 Jan 2024 18:57:29 GMT"]},"status":200}

{"level":"error","ts":1706209104.0618713,"logger":"http.handlers.reverse_proxy","msg":"aborting with incomplete response","upstream":"docker.richard:9999","duration":0.026152441,"request":{"remote_ip":"192.168.35.12","remote_port":"54610","client_ip":"192.168.35.12","proto":"HTTP/2.0","method":"GET","host":"dozzle.quesadilla.cloud","uri":"/api/events/stream","headers":{"Te":["trailers"],"X-Forwarded-Proto":["https"],"Referer":["https://dozzle.quesadilla.cloud/"],"Sec-Fetch-Site":["same-origin"],"Accept-Language":["en-US,en;q=0.5"],"Sec-Fetch-Mode":["cors"],"User-Agent":["Mozilla/5.0 (X11; Linux x86_64; rv:121.0) Gecko/20100101 Firefox/121.0"],"X-Forwarded-For":["192.168.35.12"],"Pragma":["no-cache"],"Accept-Encoding":["gzip, deflate, br"],"Sec-Fetch-Dest":["empty"],"Cache-Control":["no-cache"],"Accept":["text/event-stream"],"X-Forwarded-Host":["dozzle.quesadilla.cloud"]},"tls":{"resumed":false,"version":772,"cipher_suite":4867,"proto":"h2","server_name":"dozzle.quesadilla.cloud"}},"error":"reading: context canceled"}

{"level":"debug","ts":1706209104.0966997,"logger":"http.handlers.reverse_proxy","msg":"selected upstream","dial":"docker.richard:9999","total_upstreams":1}

{"level":"debug","ts":1706209104.1085157,"logger":"http.handlers.reverse_proxy","msg":"upstream roundtrip","upstream":"docker.richard:9999","duration":0.011444389,"request":{"remote_ip":"192.168.35.12","remote_port":"54610","client_ip":"192.168.35.12","proto":"HTTP/2.0","method":"GET","host":"dozzle.quesadilla.cloud","uri":"/","headers":{"User-Agent":["Mozilla/5.0 (X11; Linux x86_64; rv:121.0) Gecko/20100101 Firefox/121.0"],"X-Forwarded-For":["192.168.35.12"],"Accept-Encoding":["gzip, deflate, br"],"Sec-Fetch-Dest":["document"],"Sec-Fetch-Mode":["navigate"],"X-Forwarded-Host":["dozzle.quesadilla.cloud"],"Te":["trailers"],"Accept-Language":["en-US,en;q=0.5"],"Upgrade-Insecure-Requests":["1"],"Sec-Fetch-Site":["none"],"X-Forwarded-Proto":["https"],"Accept":["text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8"],"Sec-Fetch-User":["?1"]},"tls":{"resumed":false,"version":772,"cipher_suite":4867,"proto":"h2","server_name":"dozzle.quesadilla.cloud"}},"headers":{"Date":["Thu, 25 Jan 2024 18:58:24 GMT"],"Content-Type":["text/html; charset=utf-8"],"Content-Security-Policy":["default-src 'none'; script-src 'self'; style-src 'self' 'unsafe-inline'; img-src 'self' gravatar.com data:; manifest-src 'self'; connect-src 'self' api.github.com;"]},"status":200}

{"level":"debug","ts":1706209104.2550602,"logger":"http.handlers.reverse_proxy","msg":"selected upstream","dial":"docker.richard:9999","total_upstreams":1}

{"level":"debug","ts":1706209104.2585356,"logger":"http.handlers.reverse_proxy","msg":"selected upstream","dial":"docker.richard:9999","total_upstreams":1}

{"level":"debug","ts":1706209104.259652,"logger":"http.handlers.reverse_proxy","msg":"upstream roundtrip","upstream":"docker.richard:9999","duration":0.0044871,"request":{"remote_ip":"192.168.35.12","remote_port":"54610","client_ip":"192.168.35.12","proto":"HTTP/2.0","method":"GET","host":"dozzle.quesadilla.cloud","uri":"/assets/main-6433eae2.css","headers":{"Sec-Fetch-Dest":["style"],"Sec-Fetch-Site":["same-origin"],"Accept-Language":["en-US,en;q=0.5"],"Referer":["https://dozzle.quesadilla.cloud/"],"X-Forwarded-For":["192.168.35.12"],"X-Forwarded-Host":["dozzle.quesadilla.cloud"],"Sec-Fetch-Mode":["no-cors"],"Te":["trailers"],"User-Agent":["Mozilla/5.0 (X11; Linux x86_64; rv:121.0) Gecko/20100101 Firefox/121.0"],"Accept":["text/css,*/*;q=0.1"],"Accept-Encoding":["gzip, deflate, br"],"X-Forwarded-Proto":["https"]},"tls":{"resumed":false,"version":772,"cipher_suite":4867,"proto":"h2","server_name":"dozzle.quesadilla.cloud"}},"headers":{"Accept-Ranges":["bytes"],"Content-Length":["96010"],"Content-Security-Policy":["default-src 'none'; script-src 'self'; style-src 'self' 'unsafe-inline'; img-src 'self' gravatar.com data:; manifest-src 'self'; connect-src 'self' api.github.com;"],"Content-Type":["text/css; charset=utf-8"],"Date":["Thu, 25 Jan 2024 18:58:24 GMT"]},"status":200}

{"level":"debug","ts":1706209104.2663238,"logger":"http.handlers.reverse_proxy","msg":"upstream roundtrip","upstream":"docker.richard:9999","duration":0.007640065,"request":{"remote_ip":"192.168.35.12","remote_port":"54610","client_ip":"192.168.35.12","proto":"HTTP/2.0","method":"GET","host":"dozzle.quesadilla.cloud","uri":"/assets/main-a4d6d326.js","headers":{"Referer":["https://dozzle.quesadilla.cloud/"],"Sec-Fetch-Dest":["script"],"X-Forwarded-For":["192.168.35.12"],"Accept":["*/*"],"Accept-Language":["en-US,en;q=0.5"],"Sec-Fetch-Site":["same-origin"],"Te":["trailers"],"X-Forwarded-Host":["dozzle.quesadilla.cloud"],"Accept-Encoding":["gzip, deflate, br"],"User-Agent":["Mozilla/5.0 (X11; Linux x86_64; rv:121.0) Gecko/20100101 Firefox/121.0"],"Sec-Fetch-Mode":["cors"],"X-Forwarded-Proto":["https"]},"tls":{"resumed":false,"version":772,"cipher_suite":4867,"proto":"h2","server_name":"dozzle.quesadilla.cloud"}},"headers":{"Content-Length":["376098"],"Content-Security-Policy":["default-src 'none'; script-src 'self'; style-src 'self' 'unsafe-inline'; img-src 'self' gravatar.com data:; manifest-src 'self'; connect-src 'self' api.github.com;"],"Content-Type":["text/javascript; charset=utf-8"],"Date":["Thu, 25 Jan 2024 18:58:24 GMT"],"Accept-Ranges":["bytes"]},"status":200}

{"level":"debug","ts":1706209104.4003494,"logger":"http.handlers.reverse_proxy","msg":"selected upstream","dial":"docker.richard:9999","total_upstreams":1}

{"level":"debug","ts":1706209104.4156728,"logger":"http.handlers.reverse_proxy","msg":"upstream roundtrip","upstream":"docker.richard:9999","duration":0.015146195,"request":{"remote_ip":"192.168.35.12","remote_port":"54610","client_ip":"192.168.35.12","proto":"HTTP/2.0","method":"GET","host":"dozzle.quesadilla.cloud","uri":"/api/events/stream","headers":{"Accept-Language":["en-US,en;q=0.5"],"Sec-Fetch-Dest":["empty"],"X-Forwarded-For":["192.168.35.12"],"Sec-Fetch-Site":["same-origin"],"Sec-Fetch-Mode":["cors"],"Te":["trailers"],"Accept-Encoding":["gzip, deflate, br"],"Referer":["https://dozzle.quesadilla.cloud/"],"User-Agent":["Mozilla/5.0 (X11; Linux x86_64; rv:121.0) Gecko/20100101 Firefox/121.0"],"Accept":["text/event-stream"],"Pragma":["no-cache"],"Cache-Control":["no-cache"],"X-Forwarded-Proto":["https"],"X-Forwarded-Host":["dozzle.quesadilla.cloud"]},"tls":{"resumed":false,"version":772,"cipher_suite":4867,"proto":"h2","server_name":"dozzle.quesadilla.cloud"}},"headers":{"Cache-Control":["no-transform","no-cache"],"Connection":["keep-alive"],"Content-Security-Policy":["default-src 'none'; script-src 'self'; style-src 'self' 'unsafe-inline'; img-src 'self' gravatar.com data:; manifest-src 'self'; connect-src 'self' api.github.com;"],"Content-Type":["text/event-stream"],"X-Accel-Buffering":["no"],"Date":["Thu, 25 Jan 2024 18:58:24 GMT"]},"status":200}

I also tried to access over wg VPN and this is what I got. I didn’t agree to open the “unsafe” site on this side.

{"level":"debug","ts":1706210810.141591,"logger":"events","msg":"event","name":"tls_get_certificate","id":"9e3bce56-c1a9-4951-87d1-f8ad42f7212f","origin":"tls","data":{"client_hello":{"CipherSuites":[4865,4867,4866,49195,49199,52393,52392,49196,49200,49162,49161,49171,49172,156,157,47,53],"ServerName":"dozzle.quesadilla.cloud","SupportedCurves":[29,23,24,25,256,257],"SupportedPoints":"AA==","SignatureSchemes":[1027,1283,1539,2052,2053,2054,1025,1281,1537,515,513],"SupportedProtos":["h2","http/1.1"],"SupportedVersions":[772,771],"RemoteAddr":{"IP":"172.16.16.5","Port":61092,"Zone":""},"LocalAddr":{"IP":"172.20.0.8","Port":443,"Zone":""}}}}

{"level":"debug","ts":1706210810.142832,"logger":"tls.handshake","msg":"choosing certificate","identifier":"dozzle.quesadilla.cloud","num_choices":1}

{"level":"debug","ts":1706210810.1429982,"logger":"tls.handshake","msg":"default certificate selection results","identifier":"dozzle.quesadilla.cloud","subjects":["dozzle.quesadilla.cloud"],"managed":true,"issuer_key":"acme-staging-v02.api.letsencrypt.org-directory","hash":"7b0fe6a3f09d0f023a9502ed0dcf4176590c4194d7b0e5b9837c3cf7d6c74cf6"}

{"level":"debug","ts":1706210810.1430602,"logger":"tls.handshake","msg":"matched certificate in cache","remote_ip":"172.16.16.5","remote_port":"61092","subjects":["dozzle.quesadilla.cloud"],"managed":true,"expiration":1713728084,"hash":"7b0fe6a3f09d0f023a9502ed0dcf4176590c4194d7b0e5b9837c3cf7d6c74cf6"}

{"level":"debug","ts":1706210810.2651966,"logger":"http.stdlib","msg":"http: TLS handshake error from 172.16.16.5:61092: remote error: tls: unknown certificate authority"}

3. Caddy version:

v2.7.6 h1:w0NymbG2m9PcvKWsrXO6EEkY9Ru4FJK8uQbYcev1p3A=

4. How I installed and ran Caddy:

I built a custom image of alpine caddy with the following Dockerfile:

RUN xcaddy build \
    --with github.com/caddy-dns/porkbun

FROM caddy:alpine
COPY --from=builder /usr/bin/caddy /usr/bin/caddy

a. System environment:

docker on Ubuntu server

b. Command:

docker-compose up -d

c. Service/unit/compose file:

version: "3.9"

networks:
  default:
    driver: bridge

services:
  caddy:
    image: bmwagner18/caddy-porkbun:latest
    container_name: caddy
    restart: unless-stopped
    cap_add:
      - NET_ADMIN
    ports:
      - "80:80"
      - "443:443"
      - "443:443/udp"
    environment:
      - PUID=$PUID
      - PGID=$PGID
      - TZ=$TZ
      - PORKBUN_API_KEY=$PORKBUN_API_KEY
      - PORKBUN_API_SECRET_KEY=$PORKBUN_API_SECRET_KEY
    volumes:
      - $HOME_DIR/Caddyfile:/etc/caddy/Caddyfile
      - $HOME_DIR/config/caddy/site:/srv
      - $HOME_DIR/data/:/data
      - $HOME_DIR/config/:/config

# Dozzle - view logs for running docker containers
  dozzle:
    image: amir20/dozzle:latest
    container_name: dozzle
    ports:
      - 9999:8080
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock # Use Docker Socket Proxy and comment this line for improved security.
    environment:
      PUID: $PUID
      PGID: $PGID
      TZ: $TZ
    restart: unless-stopped

d. My complete Caddy config:

{
    debug
    email <my_email_here>
    acme_ca https://acme-staging-v02.api.letsencrypt.org/directory
    acme_dns porkbun {
        api_key {env.PORKBUN_API_KEY}
        api_secret_key {env.PORKBUN_API_SECRET_KEY}
    }
}

dozzle.quesadilla.cloud {
    reverse_proxy docker.richard:9999
}

5. Links to relevant resources:

This is the culprit

You’re using the staging endpoint, which is not meant for production use and only for, well, staging/testing that your configuration is correct. The certificates issued by the staging endpoints aren’t trusted because they shouldn’t. Only production certificates are trusted. Remove that line.

2 Likes

Wow. Thank you so much! Dumb mistake on my part. When I was first trying to get this thing going, I added that line and it started working so I just hadn’t touched it. I’ve been battling this for a couple of days before I swallowed my pride and asked for help…

1 Like