1. Caddy version (caddy version
):
❯ docker exec -it caddy caddy version
v2.2.1 h1:Q62GWHMtztnvyRU+KPOpw6fNfeCD3SkwH7SfT1Tgt2c=
2. How I run Caddy:
Docker, built with cloudflare DNS plugin using the following Dockerfile which I re-built today:
FROM caddy:builder AS builder
RUN xcaddy build \
--with github.com/caddy-dns/cloudflare
FROM caddy:latest
COPY --from=builder /usr/bin/caddy /usr/bin/caddy
a. System environment:
Raspberry Pi 3 and Docker
b. Command:
❯ docker-compose -f caddy/docker-compose.yml up -d
c. Service/unit/compose file:
the compose file pulls in environmental variables from secrets.env
which contains the cloudflare API token. It also pulls in more general environmental vars from ../.env
which has stuff like DOMAIN (which has the base url for my domains example.com
, DOMAIN_LOCAL (which has the local network url lan.example.com
), TIMEZONE, NETWORK (for docker network name)
version: "3.8"
services:
caddy:
image: caddy_cloudflare:1.2
container_name: caddy
hostname: caddy
env_file:
- ../.env
# Add CLOUDFLARE_API_TOKEN to secret.env
# Token permissions:
# (Zone/DNS/EDIT)
# (Zone/Zone/Read)
- ./secret.env
restart: unless-stopped
ports:
- "80:80"
- "443:443"
volumes:
- ./Caddyfile:/etc/caddy/Caddyfile:ro
- ./data:/data
- ./config:/config
networks:
default:
external:
name: $NETWORK
d. My complete Caddyfile or JSON config:
{
# General Options
debug
}
# Externally Accessible
# Sigma services
portainer.{$DOMAIN} {
reverse_proxy {$IP_SIGMA}:9000
}
home.{$DOMAIN} {
reverse_proxy {$IP_SIGMA}:8123
}
# Omega services
music.{$DOMAIN} {
reverse_proxy {$IP_OMEGA}:4533
}
watch.{$DOMAIN} {
reverse_proxy {$IP_OMEGA}:8096
}
foundry.{$DOMAIN} {
reverse_proxy {$IP_OMEGA}:30000
}
quest.{$DOMAIN} {
reverse_proxy {$IP_OMEGA}:30001
}
# Shortcodes
# Sigma Services
http://dashboard,
http://home,
http://portainer,
http://pihole,
# Dream Machine Services
http://unifigui,
http://udm,
# Omega Services
http://music,
http://watch {
redir https://{host}.{$DOMAIN_LAN}
}
# Internal
# Wildcard cert for internal services
*.{$DOMAIN_LAN} {
tls {$EMAIL_ADDRESS} {
dns cloudflare {$CLOUDFLARE_API_TOKEN}
}
# Sigma services
@heimdall host dashboard.{$DOMAIN_LAN}
handle @heimdall {
reverse_proxy {$IP_SIGMA}:8143 {
transport http {
tls_insecure_skip_verify
}
}
}
@home host home.{$DOMAIN_LAN}
handle @home {
reverse_proxy {$IP_SIGMA}:8123
}
@portainer host portainer.{$DOMAIN_LAN}
handle @portainer {
reverse_proxy {$IP_SIGMA}:9000
}
@pihole host pihole.{$DOMAIN_LAN}
handle @pihole {
reverse_proxy {$IP_SIGMA}:8080
redir / /admin
}
# Dream Machine services
@unifigui host unifigui.{$DOMAIN_LAN}
handle @unifigui {
reverse_proxy {$IP_UNIFI}:443 {
transport http {
tls_insecure_skip_verify
}
}
}
@udm host udm.{$DOMAIN_LAN}
handle @udm {
reverse_proxy {$IP_UNIFI}:443 {
transport http {
tls_insecure_skip_verify
}
}
}
# Omega services
@navidrome host music.{$DOMAIN_LAN}
handle @navidrome {
reverse_proxy {$IP_OMEGA}:4533
}
@jellyfin host watch.{$DOMAIN_LAN}
handle @jellyfin {
reverse_proxy {$IP_OMEGA}:8096
}
}
3. The problem I’m having:
Certificates for two of my domains are not renewing, oddly enough one of them did renew today just fine.
There were three domains that were up for renewal and not renewing:
I discovered that the pi (which runs pihole it another docker) had its DNS server set to itself, which for some reason was causing a problem when it tried to renew. I’ve encountered this before which is why I knew to force the pi to use 1.1.1.1 and 1.0.0.1 as its nameserver instead of the pihole instance.
After setting the DNS for the pi i restarted the caddy container and it managed to get a renewed certificate for quest.example.com
but the other two still weren’t getting certs renewed.
4. Error messages and/or full log output:
{"level":"info","ts":1640218196.3969755,"logger":"tls.cache.maintenance","msg":"certificate expires soon; queuing for renewal","identifiers":["foundry.example.com"],"remaining":79662.603035737}
{"level":"info","ts":1640218196.3972147,"logger":"tls.cache.maintenance","msg":"certificate expires soon; queuing for renewal","identifiers":["watch.example.com"],"remaining":135464.602798082}
{"level":"info","ts":1640218196.3974485,"logger":"tls.cache.maintenance","msg":"attempting certificate renewal","identifiers":["foundry.example.com"],"remaining":79662.602642041}
{"level":"info","ts":1640218196.397844,"logger":"tls.cache.maintenance","msg":"attempting certificate renewal","identifiers":["watch.example.com"],"remaining":135464.602229335}
{"level":"info","ts":1640218196.399141,"logger":"tls.renew","msg":"acquiring lock","identifier":"foundry.example.com"}
{"level":"error","ts":1640218196.3997605,"logger":"tls","msg":"job failed","error":"[foundry.example.com] EOF"}
{"level":"info","ts":1640218196.3998275,"logger":"tls.renew","msg":"acquiring lock","identifier":"watch.example.com"}
{"level":"error","ts":1640218196.4008358,"logger":"tls","msg":"job failed","error":"[watch.example.com] EOF"}
{"level":"info","ts":1640218796.370082,"logger":"tls.cache.maintenance","msg":"certificate expires soon; queuing for renewal","identifiers":["foundry.example.com"],"remaining":79062.629930069}
{"level":"info","ts":1640218796.3703017,"logger":"tls.cache.maintenance","msg":"certificate expires soon; queuing for renewal","identifiers":["watch.example.com"],"remaining":134864.629707414}
{"level":"info","ts":1640218796.3705175,"logger":"tls.cache.maintenance","msg":"attempting certificate renewal","identifiers":["watch.example.com"],"remaining":134864.629547467}
{"level":"info","ts":1640218796.3708398,"logger":"tls.cache.maintenance","msg":"attempting certificate renewal","identifiers":["foundry.example.com"],"remaining":79062.629602571}
{"level":"info","ts":1640218796.3722978,"logger":"tls.renew","msg":"acquiring lock","identifier":"watch.example.com"}
{"level":"error","ts":1640218796.3729677,"logger":"tls","msg":"job failed","error":"[watch.example.com] EOF"}
{"level":"info","ts":1640218796.3733406,"logger":"tls.renew","msg":"acquiring lock","identifier":"foundry.example.com"}
{"level":"error","ts":1640218796.3740118,"logger":"tls","msg":"job failed","error":"[foundry.example.com] EOF"}
{"level":"info","ts":1640218904.011001,"msg":"shutting down apps then terminating","signal":"SIGTERM"}
{"level":"error","ts":1640218904.0128224,"logger":"http.handlers.reverse_proxy","msg":"aborting with incomplete response","error":"context canceled"}
{"level":"error","ts":1640218904.0129435,"logger":"http.handlers.reverse_proxy","msg":"aborting with incomplete response","error":"context canceled"}
{"level":"error","ts":1640218904.0128903,"logger":"http.handlers.reverse_proxy","msg":"aborting with incomplete response","error":"context canceled"}
{"level":"error","ts":1640218904.0130873,"logger":"http.handlers.reverse_proxy","msg":"aborting with incomplete response","error":"context canceled"}
{"level":"error","ts":1640218904.0133321,"logger":"http.handlers.reverse_proxy","msg":"aborting with incomplete response","error":"context canceled"}
{"level":"error","ts":1640218904.0134284,"logger":"http.handlers.reverse_proxy","msg":"aborting with incomplete response","error":"context canceled"}
{"level":"error","ts":1640218904.013584,"logger":"http.handlers.reverse_proxy","msg":"aborting with incomplete response","error":"context canceled"}
{"level":"error","ts":1640218904.0137498,"logger":"http.handlers.reverse_proxy","msg":"aborting with incomplete response","error":"context canceled"}
{"level":"error","ts":1640218904.0138311,"logger":"http.handlers.reverse_proxy","msg":"aborting with incomplete response","error":"context canceled"}
{"level":"error","ts":1640218904.0140507,"logger":"http.handlers.reverse_proxy","msg":"aborting with incomplete response","error":"context canceled"}
{"level":"error","ts":1640218904.0140781,"logger":"http.handlers.reverse_proxy","msg":"aborting with incomplete response","error":"context canceled"}
{"level":"error","ts":1640218904.0143347,"logger":"http.handlers.reverse_proxy","msg":"aborting with incomplete response","error":"context canceled"}
{"level":"error","ts":1640218904.0143757,"logger":"http.handlers.reverse_proxy","msg":"aborting with incomplete response","error":"context canceled"}
{"level":"error","ts":1640218904.0146723,"logger":"http.handlers.reverse_proxy","msg":"aborting with incomplete response","error":"context canceled"}
{"level":"error","ts":1640218904.0147529,"logger":"http.handlers.reverse_proxy","msg":"aborting with incomplete response","error":"context canceled"}
{"level":"error","ts":1640218904.0149755,"logger":"http.handlers.reverse_proxy","msg":"aborting with incomplete response","error":"context canceled"}
{"level":"error","ts":1640218904.0151072,"logger":"http.handlers.reverse_proxy","msg":"aborting with incomplete response","error":"context canceled"}
{"level":"error","ts":1640218904.0152535,"logger":"http.handlers.reverse_proxy","msg":"aborting with incomplete response","error":"context canceled"}
{"level":"error","ts":1640218904.015413,"logger":"http.handlers.reverse_proxy","msg":"aborting with incomplete response","error":"context canceled"}
{"level":"error","ts":1640218904.0155902,"logger":"http.handlers.reverse_proxy","msg":"aborting with incomplete response","error":"context canceled"}
{"level":"error","ts":1640218904.0157785,"logger":"http.handlers.reverse_proxy","msg":"aborting with incomplete response","error":"context canceled"}
{"level":"error","ts":1640218904.0158987,"logger":"http.handlers.reverse_proxy","msg":"aborting with incomplete response","error":"context canceled"}
{"level":"error","ts":1640218904.0161397,"logger":"http.handlers.reverse_proxy","msg":"aborting with incomplete response","error":"context canceled"}
{"level":"error","ts":1640218904.0161421,"logger":"http.handlers.reverse_proxy","msg":"aborting with incomplete response","error":"context canceled"}
{"level":"error","ts":1640218904.0220942,"logger":"http.handlers.reverse_proxy","msg":"aborting with incomplete response","error":"context canceled"}
{"level":"info","ts":1640218904.5177917,"logger":"tls.cache.maintenance","msg":"stopped background certificate maintenance","cache":"0x2645860"}
{"level":"info","ts":1640218905.0184069,"logger":"admin","msg":"stopped previous server"}
{"level":"info","ts":1640218905.0185494,"msg":"shutdown done","signal":"SIGTERM"}
{"level":"info","ts":1640218908.7692099,"msg":"using provided configuration","config_file":"/etc/caddy/Caddyfile","config_adapter":"caddyfile"}
{"level":"info","ts":1640218908.7946248,"logger":"admin","msg":"admin endpoint started","address":"tcp/localhost:2019","enforce_origin":false,"origins":["127.0.0.1:2019","localhost:2019","[::1]:2019"]}
{"level":"info","ts":1640218908.7957268,"logger":"tls.cache.maintenance","msg":"started background certificate maintenance","cache":"0x287bd60"}
{"level":"info","ts":1640218908.797705,"logger":"http","msg":"server is listening only on the HTTP port, so no automatic HTTPS will be applied to this server","server_name":"srv1","http_port":80}
{"level":"info","ts":1640218908.7984715,"logger":"http","msg":"server is listening only on the HTTPS port but has no TLS connection policies; adding one to enable TLS","server_name":"srv0","https_port":443}
{"level":"info","ts":1640218908.798804,"logger":"http","msg":"enabling automatic HTTP->HTTPS redirects","server_name":"srv0"}
{"level":"warn","ts":1640218908.7992046,"logger":"http","msg":"user server is listening on same interface as automatic HTTP->HTTPS redirects; user-configured routes might override these redirects","server_name":"srv1","interface":"tcp/:80"}
{"level":"info","ts":1640218908.8084815,"logger":"http","msg":"enabling automatic TLS certificate management","domains":["*.lan.example.com","portainer.example.com","foundry.example.com","music.example.com","watch.example.com","quest.example.com","home.example.com"]}
{"level":"info","ts":1640218908.8250785,"logger":"tls","msg":"cleaned up storage units"}
{"level":"warn","ts":1640218908.9451559,"logger":"tls","msg":"stapling OCSP","error":"invalid: OCSP response for [foundry.example.com] valid after certificate expiration (-108h42m19s)"}
{"level":"info","ts":1640218908.950992,"logger":"tls.renew","msg":"acquiring lock","identifier":"foundry.example.com"}
{"level":"error","ts":1640218908.951287,"logger":"tls","msg":"job failed","error":"foundry.example.com: renewing certificate: EOF"}
{"level":"warn","ts":1640218909.0652304,"logger":"tls","msg":"stapling OCSP","error":"invalid: OCSP response for [watch.example.com] valid after certificate expiration (-109h12m17s)"}
{"level":"info","ts":1640218909.0670946,"logger":"tls.renew","msg":"acquiring lock","identifier":"watch.example.com"}
{"level":"error","ts":1640218909.0678215,"logger":"tls","msg":"job failed","error":"watch.example.com: renewing certificate: EOF"}
{"level":"info","ts":1640218909.0812786,"msg":"autosaved config","file":"/config/caddy/autosave.json"}
{"level":"info","ts":1640218909.081368,"msg":"serving initial configuration"}
{"level":"info","ts":1640219508.8064568,"logger":"tls.cache.maintenance","msg":"certificate expires soon; queuing for renewal","identifiers":["foundry.example.com"],"remaining":78350.193550059}
{"level":"info","ts":1640219508.8066475,"logger":"tls.cache.maintenance","msg":"certificate expires soon; queuing for renewal","identifiers":["watch.example.com"],"remaining":134152.193357768}
{"level":"info","ts":1640219508.8067555,"logger":"tls.cache.maintenance","msg":"attempting certificate renewal","identifiers":["foundry.example.com"],"remaining":78350.193289591}
{"level":"info","ts":1640219508.806763,"logger":"tls.cache.maintenance","msg":"attempting certificate renewal","identifiers":["watch.example.com"],"remaining":134152.193273289}
{"level":"info","ts":1640219508.8076842,"logger":"tls.renew","msg":"acquiring lock","identifier":"foundry.example.com"}
{"level":"info","ts":1640219508.8077102,"logger":"tls.renew","msg":"acquiring lock","identifier":"watch.example.com"}
{"level":"error","ts":1640219508.8080728,"logger":"tls","msg":"job failed","error":"[foundry.example.com] EOF"}
{"level":"error","ts":1640219508.8082724,"logger":"tls","msg":"job failed","error":"[watch.example.com] EOF"}
{"level":"info","ts":1640219622.966779,"msg":"shutting down apps then terminating","signal":"SIGTERM"}
{"level":"error","ts":1640219622.9682007,"logger":"http.handlers.reverse_proxy","msg":"aborting with incomplete response","error":"context canceled"}
{"level":"error","ts":1640219622.9685616,"logger":"http.handlers.reverse_proxy","msg":"aborting with incomplete response","error":"context canceled"}
{"level":"error","ts":1640219622.9685125,"logger":"http.handlers.reverse_proxy","msg":"aborting with incomplete response","error":"context canceled"}
{"level":"error","ts":1640219622.9688387,"logger":"http.handlers.reverse_proxy","msg":"aborting with incomplete response","error":"context canceled"}
{"level":"error","ts":1640219622.9689963,"logger":"http.handlers.reverse_proxy","msg":"aborting with incomplete response","error":"context canceled"}
{"level":"error","ts":1640219622.9692202,"logger":"http.handlers.reverse_proxy","msg":"aborting with incomplete response","error":"context canceled"}
{"level":"error","ts":1640219622.9693196,"logger":"http.handlers.reverse_proxy","msg":"aborting with incomplete response","error":"context canceled"}
{"level":"error","ts":1640219622.9693203,"logger":"http.handlers.reverse_proxy","msg":"aborting with incomplete response","error":"context canceled"}
{"level":"error","ts":1640219622.969554,"logger":"http.handlers.reverse_proxy","msg":"aborting with incomplete response","error":"context canceled"}
{"level":"error","ts":1640219622.969612,"logger":"http.handlers.reverse_proxy","msg":"aborting with incomplete response","error":"context canceled"}
{"level":"error","ts":1640219622.969719,"logger":"http.handlers.reverse_proxy","msg":"aborting with incomplete response","error":"context canceled"}
{"level":"error","ts":1640219622.9698625,"logger":"http.handlers.reverse_proxy","msg":"aborting with incomplete response","error":"context canceled"}
{"level":"error","ts":1640219622.970013,"logger":"http.handlers.reverse_proxy","msg":"aborting with incomplete response","error":"context canceled"}
{"level":"error","ts":1640219622.9701848,"logger":"http.handlers.reverse_proxy","msg":"aborting with incomplete response","error":"context canceled"}
{"level":"error","ts":1640219622.9702559,"logger":"http.handlers.reverse_proxy","msg":"aborting with incomplete response","error":"context canceled"}
{"level":"error","ts":1640219622.9702935,"logger":"http.handlers.reverse_proxy","msg":"aborting with incomplete response","error":"context canceled"}
{"level":"error","ts":1640219622.9704902,"logger":"http.handlers.reverse_proxy","msg":"aborting with incomplete response","error":"context canceled"}
{"level":"error","ts":1640219622.9705496,"logger":"http.handlers.reverse_proxy","msg":"aborting with incomplete response","error":"context canceled"}
{"level":"error","ts":1640219622.9707363,"logger":"http.handlers.reverse_proxy","msg":"aborting with incomplete response","error":"context canceled"}
{"level":"error","ts":1640219622.9708512,"logger":"http.handlers.reverse_proxy","msg":"aborting with incomplete response","error":"context canceled"}
{"level":"error","ts":1640219622.9711993,"logger":"http.handlers.reverse_proxy","msg":"aborting with incomplete response","error":"context canceled"}
{"level":"error","ts":1640219622.971358,"logger":"http.handlers.reverse_proxy","msg":"aborting with incomplete response","error":"context canceled"}
{"level":"error","ts":1640219622.9713426,"logger":"http.handlers.reverse_proxy","msg":"aborting with incomplete response","error":"context canceled"}
{"level":"error","ts":1640219622.9715066,"logger":"http.handlers.reverse_proxy","msg":"aborting with incomplete response","error":"context canceled"}
{"level":"info","ts":1640219623.9680617,"logger":"tls.cache.maintenance","msg":"stopped background certificate maintenance","cache":"0x287bd60"}
{"level":"info","ts":1640219624.4688842,"logger":"admin","msg":"stopped previous server"}
{"level":"info","ts":1640219624.4690835,"msg":"shutdown done","signal":"SIGTERM"}
{"level":"info","ts":1640219627.435574,"msg":"using provided configuration","config_file":"/etc/caddy/Caddyfile","config_adapter":"caddyfile"}
{"level":"info","ts":1640219627.4606447,"logger":"admin","msg":"admin endpoint started","address":"tcp/localhost:2019","enforce_origin":false,"origins":["[::1]:2019","127.0.0.1:2019","localhost:2019"]}
{"level":"info","ts":1640219627.462219,"logger":"tls.cache.maintenance","msg":"started background certificate maintenance","cache":"0x35083c0"}
{"level":"info","ts":1640219627.4631114,"logger":"http","msg":"server is listening only on the HTTPS port but has no TLS connection policies; adding one to enable TLS","server_name":"srv0","https_port":443}
{"level":"info","ts":1640219627.4632707,"logger":"http","msg":"enabling automatic HTTP->HTTPS redirects","server_name":"srv0"}
{"level":"info","ts":1640219627.4633827,"logger":"http","msg":"server is listening only on the HTTP port, so no automatic HTTPS will be applied to this server","server_name":"srv1","http_port":80}
{"level":"warn","ts":1640219627.4635437,"logger":"http","msg":"user server is listening on same interface as automatic HTTP->HTTPS redirects; user-configured routes might override these redirects","server_name":"srv1","interface":"tcp/:80"}
{"level":"debug","ts":1640219627.4724073,"logger":"http","msg":"starting server loop","address":"[::]:443","http3":false,"tls":true}
{"level":"debug","ts":1640219627.472716,"logger":"http","msg":"starting server loop","address":"[::]:80","http3":false,"tls":false}
{"level":"info","ts":1640219627.4727914,"logger":"http","msg":"enabling automatic TLS certificate management","domains":["music.example.com","watch.example.com","quest.example.com","home.example.com","*.lan.example.com","portainer.example.com","foundry.example.com"]}
{"level":"info","ts":1640219627.5103219,"logger":"tls","msg":"cleaned up storage units"}
{"level":"warn","ts":1640219627.5494103,"logger":"tls","msg":"stapling OCSP","error":"invalid: OCSP response for [watch.example.com] valid after certificate expiration (-109h12m17s)"}
{"level":"info","ts":1640219627.5552747,"logger":"tls.renew","msg":"acquiring lock","identifier":"watch.example.com"}
{"level":"error","ts":1640219627.5560215,"logger":"tls","msg":"job failed","error":"watch.example.com: renewing certificate: EOF"}
{"level":"warn","ts":1640219627.5927732,"logger":"tls","msg":"stapling OCSP","error":"invalid: OCSP response for [foundry.example.com] valid after certificate expiration (-108h42m19s)"}
{"level":"info","ts":1640219627.593731,"msg":"autosaved config","file":"/config/caddy/autosave.json"}
{"level":"info","ts":1640219627.5939062,"msg":"serving initial configuration"}
{"level":"info","ts":1640219627.5942733,"logger":"tls.renew","msg":"acquiring lock","identifier":"foundry.example.com"}
{"level":"error","ts":1640219627.5948067,"logger":"tls","msg":"job failed","error":"foundry.example.com: renewing certificate: EOF"}
5. What I already tried:
I tried re-making the API token on cloudflare, i’ve re-built caddy with cloudflare dns this morning with latest caddy.