1. The problem I’m having:
I’m running docker containers for various services such as vaultwarden and unifi controller locally that are not exposed to the internet. Caddy also runs as a docker container. My setup is described here in a previous question:. But in short I use duckdns as my domain for caddy and give it my raspberry pi private ip address to use. As far as I’m aware I havn’t changed anything in months aside from keeping my docker containers up to date and upgrading the software on my raspberry pi. My problem is now I’m no longer able to access services over https via caddy, only http works and I don’t know why.
2. Error messages and/or full log output:
caddy docker logs:
{"level":"info","ts":1740148326.8670964,"msg":"using config from file","file":"/etc/caddy/Caddyfile"}
{"level":"warn","ts":1740148326.8696938,"logger":"caddyfile","msg":"Unnecessary header_up X-Forwarded-For: the reverse proxy's default behavior is to pass headers to the upstream"}
{"level":"warn","ts":1740148326.8700013,"logger":"caddyfile","msg":"Unnecessary header_up X-Forwarded-Proto: the reverse proxy's default behavior is to pass headers to the upstream"}
{"level":"info","ts":1740148326.8736362,"msg":"adapted config to JSON","adapter":"caddyfile"}
{"level":"info","ts":1740148326.8786979,"logger":"admin","msg":"admin endpoint started","address":"localhost:2019","enforce_origin":false,"origins":["//localhost:2019","//[::1]:2019","//127.0.0.1:2019"]}
{"level":"info","ts":1740148326.8794713,"logger":"tls.cache.maintenance","msg":"started background certificate maintenance","cache":"0x40007f4480"}
{"level":"info","ts":1740148326.8796668,"logger":"http.auto_https","msg":"server is listening only on the HTTPS port but has no TLS connection policies; adding one to enable TLS","server_name":"srv0","https_port":443}
{"level":"info","ts":1740148326.8798208,"logger":"http.auto_https","msg":"enabling automatic HTTP->HTTPS redirects","server_name":"srv0"}
{"level":"info","ts":1740148326.8799458,"logger":"http.auto_https","msg":"enabling automatic HTTP->HTTPS redirects","server_name":"srv1"}
{"level":"info","ts":1740148326.8820038,"logger":"http","msg":"enabling HTTP/3 listener","addr":":443"}
{"level":"info","ts":1740148326.8828826,"msg":"failed to sufficiently increase receive buffer size (was: 208 kiB, wanted: 7168 kiB, got: 416 kiB). See https://github.com/quic-go/quic-go/wiki/UDP-Buffer-Sizes for details."}
{"level":"info","ts":1740148326.8834317,"logger":"http.log","msg":"server running","name":"srv0","protocols":["h1","h2","h3"]}
{"level":"info","ts":1740148326.8838716,"logger":"http","msg":"enabling HTTP/3 listener","addr":":8443"}
{"level":"info","ts":1740148326.8845901,"logger":"http.log","msg":"server running","name":"srv1","protocols":["h1","h2","h3"]}
{"level":"info","ts":1740148326.8851213,"logger":"http.log","msg":"server running","name":"remaining_auto_https_redirects","protocols":["h1","h2","h3"]}
{"level":"info","ts":1740148326.885365,"logger":"http","msg":"enabling automatic TLS certificate management","domains":["*.test111.duckdns.org","vaultwarden.test111.duckdns.org","unifi.test111.duckdns.org"]}
{"level":"info","ts":1740148326.8938475,"logger":"tls","msg":"cleaning storage unit","storage":"FileStorage:/data/caddy"}
{"level":"info","ts":1740148326.8960075,"logger":"tls","msg":"finished cleaning storage units"}
{"level":"warn","ts":1740148327.1861477,"logger":"tls","msg":"stapling OCSP","error":"no OCSP stapling for [*.test111.duckdns.org]: parsing OCSP response: ocsp: error from server: unauthorized","identifiers":["*.test111.duckdns.org"]}
{"level":"info","ts":1740148327.1868157,"logger":"tls","msg":"certificate needs renewal based on ARI window","subjects":["*.test111.duckdns.org"],"expiration":1739629647,"ari_cert_id":"nytfzzwhT50Et-0rLMTGcIvS1w0.BPN4SQ8K9b0Nl7k5jPBxgq02","next_ari_update":1740159090.058972,"renew_check_interval":600,"window_start":1736952416,"window_end":1737125216,"selected_time":1737015195,"renewal_cutoff":1737014595}
{"level":"info","ts":1740148327.1879928,"logger":"tls.obtain","msg":"acquiring lock","identifier":"unifi.test111.duckdns.org"}
{"level":"info","ts":1740148327.188001,"logger":"tls.obtain","msg":"acquiring lock","identifier":"vaultwarden.test111.duckdns.org"}
{"level":"info","ts":1740148327.1884358,"msg":"autosaved config (load with --resume flag)","file":"/config/caddy/autosave.json"}
{"level":"info","ts":1740148327.1885104,"msg":"serving initial configuration"}
{"level":"info","ts":1740148327.1885607,"logger":"tls.renew","msg":"acquiring lock","identifier":"*.test111.duckdns.org"}
{"level":"info","ts":1740148327.2027373,"logger":"tls.obtain","msg":"lock acquired","identifier":"unifi.test111.duckdns.org"}
{"level":"info","ts":1740148327.2027228,"logger":"tls.obtain","msg":"lock acquired","identifier":"vaultwarden.test111.duckdns.org"}
{"level":"info","ts":1740148327.2029886,"logger":"tls.obtain","msg":"obtaining certificate","identifier":"vaultwarden.test111.duckdns.org"}
{"level":"info","ts":1740148327.2029967,"logger":"tls.obtain","msg":"obtaining certificate","identifier":"unifi.test111.duckdns.org"}
{"level":"info","ts":1740148327.203668,"logger":"tls.renew","msg":"lock acquired","identifier":"*.test111.duckdns.org"}
{"level":"info","ts":1740148327.2044785,"logger":"tls.renew","msg":"renewing certificate","identifier":"*.test111.duckdns.org","remaining":-518680.204466659}
{"level":"info","ts":1740148327.2051938,"logger":"tls","msg":"waiting on internal rate limiter","identifiers":["vaultwarden.test111.duckdns.org"],"ca":"https://acme-v02.api.letsencrypt.org/directory","account":""}
{"level":"info","ts":1740148327.2056885,"logger":"tls","msg":"done waiting on internal rate limiter","identifiers":["vaultwarden.test111.duckdns.org"],"ca":"https://acme-v02.api.letsencrypt.org/directory","account":""}
{"level":"info","ts":1740148327.2061317,"logger":"tls","msg":"using ACME account","account_id":"https://acme-v02.api.letsencrypt.org/acme/acct/1972895377","account_contact":[]}
{"level":"info","ts":1740148327.2083879,"logger":"tls.issuance.acme","msg":"waiting on internal rate limiter","identifiers":["*.test111.duckdns.org"],"ca":"https://acme-v02.api.letsencrypt.org/directory","account":""}
{"level":"info","ts":1740148327.20846,"logger":"tls.issuance.acme","msg":"done waiting on internal rate limiter","identifiers":["*.test111.duckdns.org"],"ca":"https://acme-v02.api.letsencrypt.org/directory","account":""}
{"level":"info","ts":1740148327.2084987,"logger":"tls.issuance.acme","msg":"using ACME account","account_id":"https://acme-v02.api.letsencrypt.org/acme/acct/1972895377","account_contact":[]}
{"level":"info","ts":1740148327.2103298,"logger":"tls","msg":"waiting on internal rate limiter","identifiers":["unifi.test111.duckdns.org"],"ca":"https://acme-v02.api.letsencrypt.org/directory","account":""}
{"level":"info","ts":1740148327.2107663,"logger":"tls","msg":"done waiting on internal rate limiter","identifiers":["unifi.test111.duckdns.org"],"ca":"https://acme-v02.api.letsencrypt.org/directory","account":""}
{"level":"info","ts":1740148327.211124,"logger":"tls","msg":"using ACME account","account_id":"https://acme-v02.api.letsencrypt.org/acme/acct/1972895377","account_contact":[]}
{"level":"info","ts":1740148328.4949827,"logger":"tls.acme_client","msg":"trying to solve challenge","identifier":"vaultwarden.test111.duckdns.org","challenge_type":"http-01","ca":"https://acme-v02.api.letsencrypt.org/directory"}
{"level":"info","ts":1740148328.6034496,"logger":"tls.acme_client","msg":"trying to solve challenge","identifier":"unifi.test111.duckdns.org","challenge_type":"http-01","ca":"https://acme-v02.api.letsencrypt.org/directory"}
{"level":"info","ts":1740148328.714758,"logger":"tls.issuance.acme.acme_client","msg":"trying to solve challenge","identifier":"*.test111.duckdns.org","challenge_type":"dns-01","ca":"https://acme-v02.api.letsencrypt.org/directory"}
{"level":"error","ts":1740148329.8142433,"logger":"tls.acme_client","msg":"challenge failed","identifier":"vaultwarden.test111.duckdns.org","challenge_type":"http-01","problem":{"type":"urn:ietf:params:acme:error:dns","title":"","detail":"no valid A records found for vaultwarden.test111.duckdns.org; no valid AAAA records found for vaultwarden.test111.duckdns.org","instance":"","subproblems":[]}}
{"level":"error","ts":1740148329.8147397,"logger":"tls.acme_client","msg":"validating authorization","identifier":"vaultwarden.test111.duckdns.org","problem":{"type":"urn:ietf:params:acme:error:dns","title":"","detail":"no valid A records found for vaultwarden.test111.duckdns.org; no valid AAAA records found for vaultwarden.test111.duckdns.org","instance":"","subproblems":[]},"order":"https://acme-v02.api.letsencrypt.org/acme/order/1972895377/356632632765","attempt":1,"max_attempts":3}
{"level":"info","ts":1740148331.2140002,"logger":"tls.acme_client","msg":"trying to solve challenge","identifier":"vaultwarden.test111.duckdns.org","challenge_type":"tls-alpn-01","ca":"https://acme-v02.api.letsencrypt.org/directory"}
{"level":"error","ts":1740148332.71899,"logger":"tls.issuance.acme.acme_client","msg":"cleaning up solver","identifier":"*.test111.duckdns.org","challenge_type":"dns-01","error":"no memory of presenting a DNS record for \"_acme-challenge.test111.duckdns.org\" (usually OK if presenting also failed)"}
{"level":"error","ts":1740148332.93652,"logger":"tls.renew","msg":"could not get certificate from issuer","identifier":"*.test111.duckdns.org","issuer":"acme-v02.api.letsencrypt.org-directory","error":"[*.test111.duckdns.org] solving challenges: presenting for challenge: could not determine zone for domain \"_acme-challenge.test111.duckdns.org\": unexpected response code 'SERVFAIL' for _acme-challenge.test111.duckdns.org. (order=https://acme-v02.api.letsencrypt.org/acme/order/1972895377/356632633745) (ca=https://acme-v02.api.letsencrypt.org/directory)"}
{"level":"error","ts":1740148332.9367692,"logger":"tls.renew","msg":"will retry","error":"[*.test111.duckdns.org] Renew: [*.test111.duckdns.org] solving challenges: presenting for challenge: could not determine zone for domain \"_acme-challenge.test111.duckdns.org\": unexpected response code 'SERVFAIL' for _acme-challenge.test111.duckdns.org. (order=https://acme-v02.api.letsencrypt.org/acme/order/1972895377/356632633745) (ca=https://acme-v02.api.letsencrypt.org/directory)","attempt":1,"retrying_in":60,"elapsed":5.733060649,"max_duration":2592000}
{"level":"error","ts":1740148334.0452528,"logger":"tls.acme_client","msg":"challenge failed","identifier":"vaultwarden.test111.duckdns.org","challenge_type":"tls-alpn-01","problem":{"type":"urn:ietf:params:acme:error:dns","title":"","detail":"DNS problem: SERVFAIL looking up A for vaultwarden.test111.duckdns.org - the domain's nameservers may be malfunctioning; no valid AAAA records found for vaultwarden.test111.duckdns.org","instance":"","subproblems":[]}}
{"level":"error","ts":1740148334.0454059,"logger":"tls.acme_client","msg":"validating authorization","identifier":"vaultwarden.test111.duckdns.org","problem":{"type":"urn:ietf:params:acme:error:dns","title":"","detail":"DNS problem: SERVFAIL looking up A for vaultwarden.test111.duckdns.org - the domain's nameservers may be malfunctioning; no valid AAAA records found for vaultwarden.test111.duckdns.org","instance":"","subproblems":[]},"order":"https://acme-v02.api.letsencrypt.org/acme/order/1972895377/356632643375","attempt":2,"max_attempts":3}
{"level":"error","ts":1740148334.0455477,"logger":"tls.obtain","msg":"could not get certificate from issuer","identifier":"vaultwarden.test111.duckdns.org","issuer":"acme-v02.api.letsencrypt.org-directory","error":"HTTP 400 urn:ietf:params:acme:error:dns - DNS problem: SERVFAIL looking up A for vaultwarden.test111.duckdns.org - the domain's nameservers may be malfunctioning; no valid AAAA records found for vaultwarden.test111.duckdns.org"}
{"level":"error","ts":1740148334.0457563,"logger":"tls.obtain","msg":"will retry","error":"[vaultwarden.test111.duckdns.org] Obtain: [vaultwarden.test111.duckdns.org] solving challenge: vaultwarden.test111.duckdns.org: [vaultwarden.test111.duckdns.org] authorization failed: HTTP 400 urn:ietf:params:acme:error:dns - DNS problem: SERVFAIL looking up A for vaultwarden.test111.duckdns.org - the domain's nameservers may be malfunctioning; no valid AAAA records found for vaultwarden.test111.duckdns.org (ca=https://acme-v02.api.letsencrypt.org/directory)","attempt":1,"retrying_in":60,"elapsed":6.842919835,"max_duration":2592000}
{"level":"error","ts":1740148335.3036315,"logger":"tls.acme_client","msg":"challenge failed","identifier":"unifi.test111.duckdns.org","challenge_type":"http-01","problem":{"type":"urn:ietf:params:acme:error:dns","title":"","detail":"no valid A records found for unifi.test111.duckdns.org; DNS problem: SERVFAIL looking up AAAA for unifi.test111.duckdns.org - the domain's nameservers may be malfunctioning","instance":"","subproblems":[]}}
{"level":"error","ts":1740148335.3037498,"logger":"tls.acme_client","msg":"validating authorization","identifier":"unifi.test111.duckdns.org","problem":{"type":"urn:ietf:params:acme:error:dns","title":"","detail":"no valid A records found for unifi.test111.duckdns.org; DNS problem: SERVFAIL looking up AAAA for unifi.test111.duckdns.org - the domain's nameservers may be malfunctioning","instance":"","subproblems":[]},"order":"https://acme-v02.api.letsencrypt.org/acme/order/1972895377/356632633185","attempt":1,"max_attempts":3}
{"level":"info","ts":1740148336.712786,"logger":"tls.acme_client","msg":"trying to solve challenge","identifier":"unifi.test111.duckdns.org","challenge_type":"tls-alpn-01","ca":"https://acme-v02.api.letsencrypt.org/directory"}
{"level":"error","ts":1740148340.3112051,"logger":"tls.acme_client","msg":"challenge failed","identifier":"unifi.test111.duckdns.org","challenge_type":"tls-alpn-01","problem":{"type":"urn:ietf:params:acme:error:dns","title":"","detail":"no valid A records found for unifi.test111.duckdns.org; no valid AAAA records found for unifi.test111.duckdns.org","instance":"","subproblems":[]}}
{"level":"error","ts":1740148340.3113544,"logger":"tls.acme_client","msg":"validating authorization","identifier":"unifi.test111.duckdns.org","problem":{"type":"urn:ietf:params:acme:error:dns","title":"","detail":"no valid A records found for unifi.test111.duckdns.org; no valid AAAA records found for unifi.test111.duckdns.org","instance":"","subproblems":[]},"order":"https://acme-v02.api.letsencrypt.org/acme/order/1972895377/356632663335","attempt":2,"max_attempts":3}
{"level":"error","ts":1740148340.3114448,"logger":"tls.obtain","msg":"could not get certificate from issuer","identifier":"unifi.test111.duckdns.org","issuer":"acme-v02.api.letsencrypt.org-directory","error":"HTTP 400 urn:ietf:params:acme:error:dns - no valid A records found for unifi.test111.duckdns.org; no valid AAAA records found for unifi.test111.duckdns.org"}
{"level":"error","ts":1740148340.3116074,"logger":"tls.obtain","msg":"will retry","error":"[unifi.test111.duckdns.org] Obtain: [unifi.test111.duckdns.org] solving challenge: unifi.test111.duckdns.org: [unifi.test111.duckdns.org] authorization failed: HTTP 400 urn:ietf:params:acme:error:dns - no valid A records found for unifi.test111.duckdns.org; no valid AAAA records found for unifi.test111.duckdns.org (ca=https://acme-v02.api.letsencrypt.org/directory)","attempt":1,"retrying_in":60,"elapsed":13.10881792,"max_duration":2592000}
3. Caddy version:
v2.8.4 h1:q3pe0wpBj1OcHFZ3n/1nl4V4bxBrYoSoab7rL9BMYNk=
4. How I installed and ran Caddy:
docker via a compsoe file
a. System environment:
Docker containers running on a Rasberry Pi that runs rasperry pi os 6.1.21-v8+ #1642
b. Command:
I tried nslookup on my duckdns domain and it resolves to my raspberry pi private IP as expected. However interestingly If I make a typo in the vaultwarden. part of the domain name it resolves to my public wan address but gives me a SERVFAIL
which I didn’t expect. I’m not sure if this is related to my issue? The reason vaultwarden.test111.duckdns.org
works is because I added a host override for my dns resolver on my router.
nslookup vaultwarden.test111.duckdns.org
Server: 127.0.0.53
Address: 127.0.0.53#53
Non-authoritative answer:
Name: vaultwarden.test111.duckdns.org
Address: 192.168.117.10
c. Service/unit/compose file:
compose file:
services:
caddy:
container_name: caddy
image: caddy:2
restart: always
ports:
- 80:80
- 443:443
- 443:443/udp # Needed for HTTP/3.
volumes:
- /usr/local/bin/caddy:/usr/bin/caddy
- /usr/local/bin/Caddyfile:/etc/caddy/Caddyfile:ro
- caddy:/config
- caddy:/data
- /opt/docker_secrets/caddy:/run/secrets:ro
env_file:
- /opt/docker_env/caddy/caddy.env
environment:
- DUCKDNS_TOKEN=/run/secrets/duckdns_token
unifi-network-application:
container_name: unifi-network-application
image: lscr.io/linuxserver/unifi-network-application:latest
restart: unless-stopped
ports:
#- 8443:8443
- 3478:3478/udp
- 10001:10001/udp
- 8080:8080
- 1900:1900/udp #optional
#- 8843:8843 #optional
#- 8880:8880 #optional
#- 6789:6789 #optional
#- 5514:5514/udp #optional
volumes:
- unifi_controller:/config
- unifi_controller:/data
- /opt/docker_secrets/unifi:/run/secrets:ro
env_file:
- /opt/docker_env/unifi/unifi.env
environment:
- MONGO_PASS=/run/secrets/unifi_mongo_db_pass
unifi-db:
container_name: unifi-db
image: mongodb-raspberrypi4-unofficial-r7.0.0:latest
restart: unless-stopped
volumes:
- unifi_controller_db:/data/db
- ./init-mongo.js:/docker-entrypoint-initdb.d/init-mongo.js:ro
vaultwarden:
container_name: vaultwarden
image: vaultwarden/server:latest
restart: always
volumes:
- vaultwarden:/data # the path before the : can be changed
- /opt/docker_secrets/vaultwarden:/run/secrets:ro
env_file:
- /opt/docker_env/vaultwarden/vaultwarden.env
environment:
- ADMIN_TOKEN=/run/secrets/vaultwarden_admin_token_hash
volumes:
caddy:
external: true
unifi_controller:
external: true
unifi_controller_db:
external: true
vaultwarden:
external: true
d. My complete Caddy config:
*.{$DOMAIN} {
tls {
dns duckdns {$DUCKDNS_TOKEN}
}
# Logs configuration (optional, adjust as necessary)
log {
level INFO
output file {$LOG_FILE} {
roll_size 10MB
roll_keep 10
}
}
# Default reverse proxy to a generic service if no specific service matches
reverse_proxy service_default:80
}
# Vaultwarden Service
vaultwarden.{$DOMAIN} {
reverse_proxy vaultwarden:80 {
header_up X-Real-IP {http.request.remote.host}
header_up X-Forwarded-For {http.request.remote.host}
header_up X-Forwarded-Proto {scheme}
}
log {
level INFO
output file {$LOG_FILE} {
roll_size 10MB
roll_keep 10
}
format filter {
wrap json
fields {
request>uri query {
replace access_token REDACTED
}
}
}
}
}
unifi.{$DOMAIN} {
reverse_proxy unifi-network-application:8443 {
transport http {
tls_insecure_skip_verify
}
}
# Add an optional redirect rule for "http://unifi.<your-domain>"
#redir https://unifi.{$DOMAIN} permanent
log {
level INFO
output file {$LOG_FILE} {
roll_size 10MB
roll_keep 10
}
}
}
unifi.{$DOMAIN}:8443 {
redir https://unifi.{$DOMAIN} permanent
}