1. The problem I’m having:
I have configured a CA that caddy need to use to generate certificates however as discussed here caddy doesn’t generate the fullchain for the host. Using the suggested approach to listen to cert_obtained, I can successfully generate the fullchain certificate for each host. However from inside the script, I can’t let caddy reload these certs.
As you can see in the logs below, there is no indication of re-caching certificates and certificate hash is the same (when generated and when requested with openssl s_client -showcerts -connect wnaji.local:443 (wnaji.local points to the ip address of the machine in /etc/hosts locally)
I have tried visiting the website as well, it only shows the caddy-root.crt and the caddy generated cert.
I logged in to the container to check /data/caddy/certificates/local/wnaji.local/wnaji.local.crt and it is already merged with the fullchain so the script is working ok but caddy is not revalidating the certificates cache.
2. Error messages and/or full log output:
Deleted some messages from security logger (too long post)
{"level":"info","ts":1735515098.5544586,"msg":"using config from file","file":"/etc/caddy/Caddyfile"}
{"level":"info","ts":1735515098.561638,"msg":"adapted config to JSON","adapter":"caddyfile"}
{"level":"info","ts":1735515098.5658405,"logger":"tls.cache.maintenance","msg":"started background certificate maintenance","cache":"0xc0016e8280"}
{"level":"info","ts":1735515098.5715077,"logger":"http.auto_https","msg":"server is listening only on the HTTPS port but has no TLS connection policies; adding one to enable TLS","server_name":"srv0","https_port":443}
{"level":"info","ts":1735515098.5715418,"logger":"http.auto_https","msg":"enabling automatic HTTP->HTTPS redirects","server_name":"srv0"}
{"level":"warn","ts":1735515098.5715597,"logger":"http.auto_https","msg":"server is listening only on the HTTP port, so no automatic HTTPS will be applied to this server","server_name":"srv1","http_port":80}
{"level":"debug","ts":1735515098.5716536,"logger":"http.auto_https","msg":"adjusted config","tls":{"automation":{"policies":[{"subjects":["wnaji.local"]},{}]}},"http":{"servers":{"srv0":{"listen":[":443"],"routes":[{"handle":[{"handler":"subroute","routes":[{"handle":[{"REAL_HOST":"{http.request.host}","handler":"vars"},{"REAL_IP":"{http.request.remote.host}","handler":"vars"}]},{"handle":[{"REAL_HOST":"{http.request.header.X-Forwarded-Host}","handler":"vars"}],"match":[{"header":{"X-Forwarded-Host":["*"]}}]},{"handle":[{"REAL_IP":"{http.request.header.Cf-Connecting-Ip}","handler":"vars"}],"match":[{"header":{"Cf-Connecting-Ip":["*"]}}]},{"handle":[{"handler":"headers","response":{"set":{"X-Served-By":["Grafana"]}}},{"handler":"authentication","providers":{"authorizer":{"gatekeeper_name":"admins_policy","route_matcher":"*"}}}]},{"handle":[{"handler":"reverse_proxy","upstreams":[{"dial":"grafana:3000"}]}],"match":[{"header":{"Connection":["*Upgrade*"],"Upgrade":["websocket"]}}]},{"handle":[{"handler":"reverse_proxy","headers":{"request":{"set":{"Host":["{http.vars.REAL_HOST}"],"X-Real-Ip":["{http.vars.REAL_IP}"]}}},"upstreams":[{"dial":"grafana:3000"}]}]}]}],"terminal":true},{"handle":[{"handler":"subroute","routes":[{"handle":[{"handler":"subroute","routes":[{"handle":[{"handler":"subroute","routes":[{"handle":[{"handler":"authenticator","portal_name":"admin_portal","route_matcher":"*"}]}]}],"match":[{"path":["*"]}]}]}]}]}],"terminal":true},{"handle":[{"handler":"subroute","routes":[{"handle":[{"REAL_HOST":"{http.request.host}","handler":"vars"},{"REAL_IP":"{http.request.remote.host}","handler":"vars"}]},{"handle":[{"REAL_HOST":"{http.request.header.X-Forwarded-Host}","handler":"vars"}],"match":[{"header":{"X-Forwarded-Host":["*"]}}]},{"handle":[{"REAL_IP":"{http.request.header.Cf-Connecting-Ip}","handler":"vars"}],"match":[{"header":{"Cf-Connecting-Ip":["*"]}}]},{"handle":[{"handler":"headers","response":{"set":{"X-Served-By":["Loki"]}}},{"handler":"authentication","providers":{"authorizer":{"gatekeeper_name":"admins_policy","route_matcher":"*"}}}]},{"handle":[{"handler":"reverse_proxy","upstreams":[{"dial":"loki:3100"}]}],"match":[{"header":{"Connection":["*Upgrade*"],"Upgrade":["websocket"]}}]},{"handle":[{"handler":"reverse_proxy","headers":{"request":{"set":{"Host":["{http.vars.REAL_HOST}"],"X-Real-Ip":["{http.vars.REAL_IP}"]}}},"upstreams":[{"dial":"loki:3100"}]}]}]}],"terminal":true},{"handle":[{"handler":"subroute","routes":[{"handle":[{"REAL_HOST":"{http.request.host}","handler":"vars"},{"REAL_IP":"{http.request.remote.host}","handler":"vars"}]},{"handle":[{"REAL_HOST":"{http.request.header.X-Forwarded-Host}","handler":"vars"}],"match":[{"header":{"X-Forwarded-Host":["*"]}}]},{"handle":[{"REAL_IP":"{http.request.header.Cf-Connecting-Ip}","handler":"vars"}],"match":[{"header":{"Cf-Connecting-Ip":["*"]}}]},{"handle":[{"handler":"headers","response":{"set":{"X-Served-By":["promtail"]}}},{"handler":"authentication","providers":{"authorizer":{"gatekeeper_name":"admins_policy","route_matcher":"*"}}}]},{"handle":[{"handler":"reverse_proxy","upstreams":[{"dial":"promtail:9080"}]}],"match":[{"header":{"Connection":["*Upgrade*"],"Upgrade":["websocket"]}}]},{"handle":[{"handler":"reverse_proxy","headers":{"request":{"set":{"Host":["{http.vars.REAL_HOST}"],"X-Real-Ip":["{http.vars.REAL_IP}"]}}},"upstreams":[{"dial":"promtail:9080"}]}]}]}],"terminal":true},{"handle":[{"handler":"subroute","routes":[{"handle":[{"REAL_HOST":"{http.request.host}","handler":"vars"},{"REAL_IP":"{http.request.remote.host}","handler":"vars"}]},{"handle":[{"REAL_HOST":"{http.request.header.X-Forwarded-Host}","handler":"vars"}],"match":[{"header":{"X-Forwarded-Host":["*"]}}]},{"handle":[{"REAL_IP":"{http.request.header.Cf-Connecting-Ip}","handler":"vars"}],"match":[{"header":{"Cf-Connecting-Ip":["*"]}}]},{"handle":[{"handler":"headers","response":{"set":{"X-Served-By":["Alertmanager"]}}},{"handler":"authentication","providers":{"authorizer":{"gatekeeper_name":"admins_policy","route_matcher":"*"}}}]},{"handle":[{"handler":"reverse_proxy","upstreams":[{"dial":"alertmanager:9093"}]}],"match":[{"header":{"Connection":["*Upgrade*"],"Upgrade":["websocket"]}}]},{"handle":[{"handler":"reverse_proxy","headers":{"request":{"set":{"Host":["{http.vars.REAL_HOST}"],"X-Real-Ip":["{http.vars.REAL_IP}"]}}},"upstreams":[{"dial":"alertmanager:9093"}]}]}]}],"terminal":true},{"handle":[{"handler":"subroute","routes":[{"handle":[{"REAL_HOST":"{http.request.host}","handler":"vars"},{"REAL_IP":"{http.request.remote.host}","handler":"vars"}]},{"handle":[{"REAL_HOST":"{http.request.header.X-Forwarded-Host}","handler":"vars"}],"match":[{"header":{"X-Forwarded-Host":["*"]}}]},{"handle":[{"REAL_IP":"{http.request.header.Cf-Connecting-Ip}","handler":"vars"}],"match":[{"header":{"Cf-Connecting-Ip":["*"]}}]},{"handle":[{"handler":"headers","response":{"set":{"X-Served-By":["Prometheus"]}}},{"handler":"authentication","providers":{"authorizer":{"gatekeeper_name":"admins_policy","route_matcher":"*"}}}]},{"handle":[{"handler":"reverse_proxy","upstreams":[{"dial":"prometheus:9090"}]}],"match":[{"header":{"Connection":["*Upgrade*"],"Upgrade":["websocket"]}}]},{"handle":[{"handler":"reverse_proxy","headers":{"request":{"set":{"Host":["{http.vars.REAL_HOST}"],"X-Real-Ip":["{http.vars.REAL_IP}"]}}},"upstreams":[{"dial":"prometheus:9090"}]}]}]}],"terminal":true},{"handle":[{"handler":"subroute","routes":[{"handle":[{"REAL_HOST":"{http.request.host}","handler":"vars"},{"REAL_IP":"{http.request.remote.host}","handler":"vars"}]},{"handle":[{"REAL_HOST":"{http.request.header.X-Forwarded-Host}","handler":"vars"}],"match":[{"header":{"X-Forwarded-Host":["*"]}}]},{"handle":[{"REAL_IP":"{http.request.header.Cf-Connecting-Ip}","handler":"vars"}],"match":[{"header":{"Cf-Connecting-Ip":["*"]}}]},{"handle":[{"encodings":{"gzip":{}},"handler":"encode","prefer":["gzip"]}]},{"group":"group9","handle":[{"handler":"subroute","routes":[{"handle":[{"handler":"vars","root":"/usr/share/caddy"},{"handler":"headers","response":{"set":{"X-Served-By":["Caddy"]}}},{"handler":"file_server","hide":["/etc/caddy/sites-enabled/www.caddyfile","/etc/caddy/snippets/snippets_main.caddyfile"]}]}]},{"handler":"subroute","routes":[{"handle":[{"body":"Not found","handler":"static_response","status_code":404}]}]}]}]}],"terminal":true}],"tls_connection_policies":[{}],"automatic_https":{"prefer_wildcard":true},"trusted_proxies":{"ranges":["192.168.0.0/16","172.16.0.0/12","10.0.0.0/8","127.0.0.1/8","fd00::/8","::1"],"source":"static"},"client_ip_headers":["Cf-Connecting-Ip","X-Real-IP"],"logs":{"logger_names":{"admin.wnaji.dev":["log0"],"wnaji.dev":[""],"wnaji.local":[""]},"skip_hosts":["am.wnaji.dev","grafana.wnaji.dev","loki.wnaji.dev","pm.wnaji.dev","pmt.wnaji.dev"]}},"srv1":{"listen":[":80"],"routes":[{"handle":[{"handler":"subroute","routes":[{"handle":[{"abort":true,"handler":"static_response"}],"match":[{"not":[{"client_ip":{"ranges":["192.168.0.0/16","172.16.0.0/12","10.0.0.0/8","127.0.0.1/8","fd00::/8","::1"]}}]}]},{"handle":[{"handler":"metrics"}]}]}],"terminal":true},{},{}],"automatic_https":{"disable":true,"prefer_wildcard":true},"trusted_proxies":{"ranges":["192.168.0.0/16","172.16.0.0/12","10.0.0.0/8","127.0.0.1/8","fd00::/8","::1"],"source":"static"},"client_ip_headers":["Cf-Connecting-Ip","X-Real-IP"]}},"metrics":{}}}
{"level":"warn","ts":1735515098.6594896,"logger":"pki.ca.local","msg":"installing root certificate (you might be prompted for password)","path":"storage:pki/authorities/local/root.crt"}
{"level":"info","ts":1735515098.6597884,"msg":"warning: \"certutil\" is not available, install \"certutil\" with \"apt install libnss3-tools\" or \"yum install nss-tools\" and try again"}
{"level":"info","ts":1735515098.6598015,"msg":"define JAVA_HOME environment variable to use the Java trust"}
{"level":"info","ts":1735515098.6889987,"msg":"certificate installed properly in linux trusts"}
{"level":"debug","ts":1735515098.6891181,"logger":"security","msg":"started app instance","app":"security"}
{"level":"info","ts":1735515098.6891527,"logger":"http","msg":"enabling HTTP/3 listener","addr":":443"}
{"level":"info","ts":1735515098.6901045,"msg":"failed to sufficiently increase receive buffer size (was: 208 kiB, wanted: 7168 kiB, got: 416 kiB). See https://github.com/quic-go/quic-go/wiki/UDP-Buffer-Sizes for details."}
{"level":"debug","ts":1735515098.690417,"logger":"http","msg":"starting server loop","address":"[::]:443","tls":true,"http3":true}
{"level":"info","ts":1735515098.6904392,"logger":"http.log","msg":"server running","name":"srv0","protocols":["h1","h2","h3"]}
{"level":"warn","ts":1735515098.6904752,"logger":"http","msg":"HTTP/3 skipped because it requires TLS","network":"tcp","addr":":80"}
{"level":"debug","ts":1735515098.6905017,"logger":"http","msg":"starting server loop","address":"[::]:80","tls":false,"http3":false}
{"level":"warn","ts":1735515098.690506,"logger":"http","msg":"HTTP/2 skipped because it requires TLS","network":"tcp","addr":":80"}
{"level":"info","ts":1735515098.6905122,"logger":"http.log","msg":"server running","name":"srv1","protocols":["h1","h2","h3"]}
{"level":"info","ts":1735515098.690524,"logger":"http","msg":"enabling automatic TLS certificate management","domains":["pm.wnaji.dev","wnaji.dev","wnaji.local","grafana.wnaji.dev","admin.wnaji.dev","loki.wnaji.dev","pmt.wnaji.dev","am.wnaji.dev"]}
{"level":"info","ts":1735515098.691035,"msg":"autosaved config (load with --resume flag)","file":"/config/caddy/autosave.json"}
{"level":"info","ts":1735515098.6910675,"msg":"serving initial configuration"}
{"level":"info","ts":1735515098.694189,"logger":"tls","msg":"cleaning storage unit","storage":"FileStorage:/data/caddy"}
{"level":"info","ts":1735515098.6943884,"logger":"tls.obtain","msg":"acquiring lock","identifier":"pm.wnaji.dev"}
{"level":"info","ts":1735515098.6964662,"logger":"tls","msg":"finished cleaning storage units"}
{"level":"info","ts":1735515098.6978447,"logger":"tls.obtain","msg":"acquiring lock","identifier":"grafana.wnaji.dev"}
{"level":"info","ts":1735515098.6991196,"logger":"tls.obtain","msg":"lock acquired","identifier":"grafana.wnaji.dev"}
{"level":"info","ts":1735515098.6991835,"logger":"tls.obtain","msg":"obtaining certificate","identifier":"grafana.wnaji.dev"}
{"level":"debug","ts":1735515098.6992366,"logger":"events","msg":"event","name":"cert_obtaining","id":"058ba20d-a047-41b7-b0c6-1192525e6161","origin":"tls","data":{"identifier":"grafana.wnaji.dev"}}
{"level":"debug","ts":1735515098.700849,"logger":"tls","msg":"created CSR","identifiers":["grafana.wnaji.dev"],"san_dns_names":["grafana.wnaji.dev"],"san_emails":[],"common_name":"","extra_extensions":0}
{"level":"debug","ts":1735515098.7010946,"logger":"tls.obtain","msg":"trying issuer 1/1","issuer":"local"}
{"level":"info","ts":1735515098.7018552,"logger":"tls.obtain","msg":"lock acquired","identifier":"pm.wnaji.dev"}
{"level":"info","ts":1735515098.7020116,"logger":"tls.obtain","msg":"obtaining certificate","identifier":"pm.wnaji.dev"}
{"level":"debug","ts":1735515098.7021255,"logger":"events","msg":"event","name":"cert_obtaining","id":"c889154b-dd90-4f09-a460-5530e0c1529c","origin":"tls","data":{"identifier":"pm.wnaji.dev"}}
{"level":"debug","ts":1735515098.7022774,"logger":"tls","msg":"created CSR","identifiers":["pm.wnaji.dev"],"san_dns_names":["pm.wnaji.dev"],"san_emails":[],"common_name":"","extra_extensions":0}
{"level":"debug","ts":1735515098.7028453,"logger":"tls.obtain","msg":"trying issuer 1/1","issuer":"local"}
{"level":"debug","ts":1735515098.7039602,"logger":"pki.ca.local","msg":"using intermediate signer","serial":"682604251874200514516414899705486030998329637113","not_before":"2024-12-29 02:34:13 +0000 UTC","not_after":"2026-12-29 02:34:13 +0000 UTC"}
{"level":"info","ts":1735515098.7086887,"logger":"tls.obtain","msg":"acquiring lock","identifier":"admin.wnaji.dev"}
{"level":"info","ts":1735515098.7093925,"logger":"tls.obtain","msg":"acquiring lock","identifier":"am.wnaji.dev"}
{"level":"info","ts":1735515098.7096665,"logger":"tls.obtain","msg":"lock acquired","identifier":"admin.wnaji.dev"}
{"level":"info","ts":1735515098.712084,"logger":"tls.obtain","msg":"obtaining certificate","identifier":"admin.wnaji.dev"}
{"level":"debug","ts":1735515098.7124054,"logger":"events","msg":"event","name":"cert_obtaining","id":"7c76954c-0a74-4adb-8e07-e71e0d8d36e2","origin":"tls","data":{"identifier":"admin.wnaji.dev"}}
{"level":"debug","ts":1735515098.713924,"logger":"tls","msg":"created CSR","identifiers":["admin.wnaji.dev"],"san_dns_names":["admin.wnaji.dev"],"san_emails":[],"common_name":"","extra_extensions":0}
{"level":"debug","ts":1735515098.7142751,"logger":"tls.obtain","msg":"trying issuer 1/1","issuer":"local"}
{"level":"info","ts":1735515098.7145271,"logger":"tls.obtain","msg":"acquiring lock","identifier":"wnaji.dev"}
{"level":"info","ts":1735515098.7146885,"logger":"tls.obtain","msg":"acquiring lock","identifier":"pmt.wnaji.dev"}
{"level":"info","ts":1735515098.7152116,"logger":"tls.obtain","msg":"lock acquired","identifier":"wnaji.dev"}
{"level":"info","ts":1735515098.7152731,"logger":"tls.obtain","msg":"obtaining certificate","identifier":"wnaji.dev"}
{"level":"debug","ts":1735515098.7152932,"logger":"events","msg":"event","name":"cert_obtaining","id":"9ae843bf-feca-45c4-a157-df28ebfd64a9","origin":"tls","data":{"identifier":"wnaji.dev"}}
{"level":"debug","ts":1735515098.7153718,"logger":"tls","msg":"created CSR","identifiers":["wnaji.dev"],"san_dns_names":["wnaji.dev"],"san_emails":[],"common_name":"","extra_extensions":0}
{"level":"debug","ts":1735515098.7154784,"logger":"pki.ca.local","msg":"using intermediate signer","serial":"682604251874200514516414899705486030998329637113","not_before":"2024-12-29 02:34:13 +0000 UTC","not_after":"2026-12-29 02:34:13 +0000 UTC"}
{"level":"info","ts":1735515098.7160797,"logger":"tls.obtain","msg":"lock acquired","identifier":"am.wnaji.dev"}
{"level":"info","ts":1735515098.7162423,"logger":"tls.obtain","msg":"obtaining certificate","identifier":"am.wnaji.dev"}
{"level":"debug","ts":1735515098.7163312,"logger":"events","msg":"event","name":"cert_obtaining","id":"0421a4af-7646-44b9-964e-1e0b57632610","origin":"tls","data":{"identifier":"am.wnaji.dev"}}
{"level":"debug","ts":1735515098.7165008,"logger":"tls","msg":"created CSR","identifiers":["am.wnaji.dev"],"san_dns_names":["am.wnaji.dev"],"san_emails":[],"common_name":"","extra_extensions":0}
{"level":"debug","ts":1735515098.716824,"logger":"tls.obtain","msg":"trying issuer 1/1","issuer":"local"}
{"level":"debug","ts":1735515098.7171764,"logger":"pki.ca.local","msg":"using intermediate signer","serial":"682604251874200514516414899705486030998329637113","not_before":"2024-12-29 02:34:13 +0000 UTC","not_after":"2026-12-29 02:34:13 +0000 UTC"}
{"level":"info","ts":1735515098.715971,"logger":"tls.obtain","msg":"acquiring lock","identifier":"loki.wnaji.dev"}
{"level":"info","ts":1735515098.7138557,"logger":"tls.obtain","msg":"certificate obtained successfully","identifier":"grafana.wnaji.dev","issuer":"local"}
{"level":"debug","ts":1735515098.720829,"logger":"events","msg":"event","name":"cert_obtained","id":"92633ad1-64dd-4150-a652-bdeb92b504c7","origin":"tls","data":{"certificate_path":"certificates/local/grafana.wnaji.dev/grafana.wnaji.dev.crt","csr_pem":"###","identifier":"grafana.wnaji.dev","issuer":"local","metadata_path":"certificates/local/grafana.wnaji.dev/grafana.wnaji.dev.json","private_key_path":"certificates/local/grafana.wnaji.dev/grafana.wnaji.dev.key","renewal":false,"storage_path":"certificates/local/grafana.wnaji.dev"}}
{"level":"debug","ts":1735515098.7208693,"logger":"events","msg":"invoking subscribed handler","name":"cert_obtained","id":"92633ad1-64dd-4150-a652-bdeb92b504c7","origin":"tls","data":{"certificate_path":"certificates/local/grafana.wnaji.dev/grafana.wnaji.dev.crt","csr_pem":"###","identifier":"grafana.wnaji.dev","issuer":"local","metadata_path":"certificates/local/grafana.wnaji.dev/grafana.wnaji.dev.json","private_key_path":"certificates/local/grafana.wnaji.dev/grafana.wnaji.dev.key","renewal":false,"storage_path":"certificates/local/grafana.wnaji.dev"},"subscribed_to":"cert_obtained","handler":{"command":"sh","args":["-c","/scripts/cert_obtained.sh {event.data.certificate_path}"],"timeout":30000000000}}
{"level":"info","ts":1735515098.7209432,"logger":"tls.obtain","msg":"releasing lock","identifier":"grafana.wnaji.dev"}
{"level":"warn","ts":1735515098.7212646,"logger":"tls","msg":"stapling OCSP","error":"no OCSP stapling for [grafana.wnaji.dev]: no OCSP server specified in certificate","identifiers":["grafana.wnaji.dev"]}
{"level":"debug","ts":1735515098.7212894,"logger":"tls.cache","msg":"added certificate to cache","subjects":["grafana.wnaji.dev"],"expiration":1735558299,"managed":true,"issuer_key":"local","hash":"fc294fe6beb57f5d3921d46190b4665bff035e475259418f16c33cbbae781702","cache_size":1,"cache_capacity":10000}
{"level":"debug","ts":1735515098.7213044,"logger":"events","msg":"event","name":"cached_managed_cert","id":"1d276a0a-6140-4c68-b9d1-ee7f0977bdd0","origin":"tls","data":{"sans":["grafana.wnaji.dev"]}}
{"level":"debug","ts":1735515098.7155623,"logger":"tls.obtain","msg":"trying issuer 1/1","issuer":"local"}
{"level":"debug","ts":1735515098.7221806,"logger":"pki.ca.local","msg":"using intermediate signer","serial":"682604251874200514516414899705486030998329637113","not_before":"2024-12-29 02:34:13 +0000 UTC","not_after":"2026-12-29 02:34:13 +0000 UTC"}
{"level":"info","ts":1735515098.7138119,"logger":"tls.obtain","msg":"acquiring lock","identifier":"wnaji.local"}
{"level":"debug","ts":1735515098.715853,"logger":"pki.ca.local","msg":"using intermediate signer","serial":"682604251874200514516414899705486030998329637113","not_before":"2024-12-29 02:34:13 +0000 UTC","not_after":"2026-12-29 02:34:13 +0000 UTC"}
{"level":"info","ts":1735515098.7241318,"logger":"tls.obtain","msg":"lock acquired","identifier":"pmt.wnaji.dev"}
{"level":"info","ts":1735515098.729216,"logger":"tls.obtain","msg":"obtaining certificate","identifier":"pmt.wnaji.dev"}
{"level":"debug","ts":1735515098.7297223,"logger":"events","msg":"event","name":"cert_obtaining","id":"480a5e42-c341-4d82-ae3b-f89e34774d13","origin":"tls","data":{"identifier":"pmt.wnaji.dev"}}
{"level":"debug","ts":1735515098.7300532,"logger":"tls","msg":"created CSR","identifiers":["pmt.wnaji.dev"],"san_dns_names":["pmt.wnaji.dev"],"san_emails":[],"common_name":"","extra_extensions":0}
{"level":"debug","ts":1735515098.7307122,"logger":"tls.obtain","msg":"trying issuer 1/1","issuer":"local"}
{"level":"debug","ts":1735515098.7311044,"logger":"pki.ca.local","msg":"using intermediate signer","serial":"682604251874200514516414899705486030998329637113","not_before":"2024-12-29 02:34:13 +0000 UTC","not_after":"2026-12-29 02:34:13 +0000 UTC"}
{"level":"info","ts":1735515098.732996,"logger":"tls.obtain","msg":"lock acquired","identifier":"wnaji.local"}
{"level":"info","ts":1735515098.7332463,"logger":"tls.obtain","msg":"obtaining certificate","identifier":"wnaji.local"}
{"level":"debug","ts":1735515098.7333398,"logger":"events","msg":"event","name":"cert_obtaining","id":"84e9ea64-4912-458c-8894-3c25d75ba2cc","origin":"tls","data":{"identifier":"wnaji.local"}}
{"level":"debug","ts":1735515098.7335832,"logger":"tls","msg":"created CSR","identifiers":["wnaji.local"],"san_dns_names":["wnaji.local"],"san_emails":[],"common_name":"","extra_extensions":0}
{"level":"debug","ts":1735515098.7348685,"logger":"tls.obtain","msg":"trying issuer 1/1","issuer":"local"}
{"level":"debug","ts":1735515098.7364373,"logger":"pki.ca.local","msg":"using intermediate signer","serial":"682604251874200514516414899705486030998329637113","not_before":"2024-12-29 02:34:13 +0000 UTC","not_after":"2026-12-29 02:34:13 +0000 UTC"}
{"level":"info","ts":1735515098.7369153,"logger":"tls.obtain","msg":"lock acquired","identifier":"loki.wnaji.dev"}
{"level":"info","ts":1735515098.737366,"logger":"tls.obtain","msg":"obtaining certificate","identifier":"loki.wnaji.dev"}
{"level":"debug","ts":1735515098.7376676,"logger":"events","msg":"event","name":"cert_obtaining","id":"0915fd95-f989-44ad-8398-c0bde772ee0f","origin":"tls","data":{"identifier":"loki.wnaji.dev"}}
{"level":"debug","ts":1735515098.7381551,"logger":"tls","msg":"created CSR","identifiers":["loki.wnaji.dev"],"san_dns_names":["loki.wnaji.dev"],"san_emails":[],"common_name":"","extra_extensions":0}
{"level":"debug","ts":1735515098.7390196,"logger":"tls.obtain","msg":"trying issuer 1/1","issuer":"local"}
{"level":"debug","ts":1735515098.7397146,"logger":"pki.ca.local","msg":"using intermediate signer","serial":"682604251874200514516414899705486030998329637113","not_before":"2024-12-29 02:34:13 +0000 UTC","not_after":"2026-12-29 02:34:13 +0000 UTC"}
{"level":"info","ts":1735515098.7434256,"logger":"tls.obtain","msg":"certificate obtained successfully","identifier":"admin.wnaji.dev","issuer":"local"}
{"level":"debug","ts":1735515098.7436337,"logger":"events","msg":"event","name":"cert_obtained","id":"f90d4b74-9175-4c31-bd66-1115527a8dc2","origin":"tls","data":{"certificate_path":"certificates/local/admin.wnaji.dev/admin.wnaji.dev.crt","csr_pem":"###","identifier":"admin.wnaji.dev","issuer":"local","metadata_path":"certificates/local/admin.wnaji.dev/admin.wnaji.dev.json","private_key_path":"certificates/local/admin.wnaji.dev/admin.wnaji.dev.key","renewal":false,"storage_path":"certificates/local/admin.wnaji.dev"}}
{"level":"debug","ts":1735515098.7437687,"logger":"events","msg":"invoking subscribed handler","name":"cert_obtained","id":"f90d4b74-9175-4c31-bd66-1115527a8dc2","origin":"tls","data":{"certificate_path":"certificates/local/admin.wnaji.dev/admin.wnaji.dev.crt","csr_pem":"###","identifier":"admin.wnaji.dev","issuer":"local","metadata_path":"certificates/local/admin.wnaji.dev/admin.wnaji.dev.json","private_key_path":"certificates/local/admin.wnaji.dev/admin.wnaji.dev.key","renewal":false,"storage_path":"certificates/local/admin.wnaji.dev"},"subscribed_to":"cert_obtained","handler":{"command":"sh","args":["-c","/scripts/cert_obtained.sh {event.data.certificate_path}"],"timeout":30000000000}}
{"level":"info","ts":1735515098.7439406,"logger":"tls.obtain","msg":"releasing lock","identifier":"admin.wnaji.dev"}
{"level":"info","ts":1735515098.7379575,"logger":"tls.obtain","msg":"certificate obtained successfully","identifier":"am.wnaji.dev","issuer":"local"}
{"level":"debug","ts":1735515098.7502737,"logger":"events","msg":"event","name":"cert_obtained","id":"727d6db0-531a-4cda-89d0-9884cdd46d75","origin":"tls","data":{"certificate_path":"certificates/local/am.wnaji.dev/am.wnaji.dev.crt","csr_pem":"###","identifier":"am.wnaji.dev","issuer":"local","metadata_path":"certificates/local/am.wnaji.dev/am.wnaji.dev.json","private_key_path":"certificates/local/am.wnaji.dev/am.wnaji.dev.key","renewal":false,"storage_path":"certificates/local/am.wnaji.dev"}}
{"level":"debug","ts":1735515098.750474,"logger":"events","msg":"invoking subscribed handler","name":"cert_obtained","id":"727d6db0-531a-4cda-89d0-9884cdd46d75","origin":"tls","data":{"certificate_path":"certificates/local/am.wnaji.dev/am.wnaji.dev.crt","csr_pem":"###","identifier":"am.wnaji.dev","issuer":"local","metadata_path":"certificates/local/am.wnaji.dev/am.wnaji.dev.json","private_key_path":"certificates/local/am.wnaji.dev/am.wnaji.dev.key","renewal":false,"storage_path":"certificates/local/am.wnaji.dev"},"subscribed_to":"cert_obtained","handler":{"command":"sh","args":["-c","/scripts/cert_obtained.sh {event.data.certificate_path}"],"timeout":30000000000}}
{"level":"info","ts":1735515098.750621,"logger":"tls.obtain","msg":"releasing lock","identifier":"am.wnaji.dev"}
{"level":"warn","ts":1735515098.751374,"logger":"tls","msg":"stapling OCSP","error":"no OCSP stapling for [am.wnaji.dev]: no OCSP server specified in certificate","identifiers":["am.wnaji.dev"]}
{"level":"debug","ts":1735515098.7515595,"logger":"tls.cache","msg":"added certificate to cache","subjects":["am.wnaji.dev"],"expiration":1735558299,"managed":true,"issuer_key":"local","hash":"ce6423a991d66d90934d59725ae06a0c63e4637cbc831fd9b8203510d4a434d4","cache_size":2,"cache_capacity":10000}
{"level":"debug","ts":1735515098.7517345,"logger":"events","msg":"event","name":"cached_managed_cert","id":"fcf2de67-68b5-4e9c-9ba2-d8e94f6cecea","origin":"tls","data":{"sans":["am.wnaji.dev"]}}
{"level":"info","ts":1735515098.7520258,"logger":"tls.obtain","msg":"certificate obtained successfully","identifier":"loki.wnaji.dev","issuer":"local"}
{"level":"debug","ts":1735515098.7522721,"logger":"events","msg":"event","name":"cert_obtained","id":"4af08d30-1cc2-4b43-b58a-322cf9fb45d1","origin":"tls","data":{"certificate_path":"certificates/local/loki.wnaji.dev/loki.wnaji.dev.crt","csr_pem":"###","identifier":"loki.wnaji.dev","issuer":"local","metadata_path":"certificates/local/loki.wnaji.dev/loki.wnaji.dev.json","private_key_path":"certificates/local/loki.wnaji.dev/loki.wnaji.dev.key","renewal":false,"storage_path":"certificates/local/loki.wnaji.dev"}}
{"level":"debug","ts":1735515098.7523887,"logger":"events","msg":"invoking subscribed handler","name":"cert_obtained","id":"4af08d30-1cc2-4b43-b58a-322cf9fb45d1","origin":"tls","data":{"certificate_path":"certificates/local/loki.wnaji.dev/loki.wnaji.dev.crt","csr_pem":"###","identifier":"loki.wnaji.dev","issuer":"local","metadata_path":"certificates/local/loki.wnaji.dev/loki.wnaji.dev.json","private_key_path":"certificates/local/loki.wnaji.dev/loki.wnaji.dev.key","renewal":false,"storage_path":"certificates/local/loki.wnaji.dev"},"subscribed_to":"cert_obtained","handler":{"command":"sh","args":["-c","/scripts/cert_obtained.sh {event.data.certificate_path}"],"timeout":30000000000}}
{"level":"info","ts":1735515098.752579,"logger":"tls.obtain","msg":"releasing lock","identifier":"loki.wnaji.dev"}
{"level":"warn","ts":1735515098.7529848,"logger":"tls","msg":"stapling OCSP","error":"no OCSP stapling for [loki.wnaji.dev]: no OCSP server specified in certificate","identifiers":["loki.wnaji.dev"]}
{"level":"debug","ts":1735515098.7534316,"logger":"tls.cache","msg":"added certificate to cache","subjects":["loki.wnaji.dev"],"expiration":1735558299,"managed":true,"issuer_key":"local","hash":"a0d97004b60e9e9b438851375072a8bc376085f867ff18dc9fae25f8752e7fd1","cache_size":3,"cache_capacity":10000}
{"level":"debug","ts":1735515098.7535474,"logger":"events","msg":"event","name":"cached_managed_cert","id":"5c72caa7-46b8-4fe5-a3fd-1ead9c611d60","origin":"tls","data":{"sans":["loki.wnaji.dev"]}}
{"level":"warn","ts":1735515098.7501154,"logger":"tls","msg":"stapling OCSP","error":"no OCSP stapling for [admin.wnaji.dev]: no OCSP server specified in certificate","identifiers":["admin.wnaji.dev"]}
{"level":"debug","ts":1735515098.7537384,"logger":"tls.cache","msg":"added certificate to cache","subjects":["admin.wnaji.dev"],"expiration":1735558299,"managed":true,"issuer_key":"local","hash":"1d647578f39d34e9cc882d90885d9ea22fd75a40746a1706c2d5c834dbf7e263","cache_size":4,"cache_capacity":10000}
{"level":"debug","ts":1735515098.753839,"logger":"events","msg":"event","name":"cached_managed_cert","id":"ef088575-51e7-4df8-9069-3734c3afc81b","origin":"tls","data":{"sans":["admin.wnaji.dev"]}}
{"level":"info","ts":1735515098.7492619,"logger":"tls.obtain","msg":"certificate obtained successfully","identifier":"wnaji.dev","issuer":"local"}
{"level":"debug","ts":1735515098.755704,"logger":"events","msg":"event","name":"cert_obtained","id":"ee815a16-1ec1-40b1-bce9-4ab3f9bfd451","origin":"tls","data":{"certificate_path":"certificates/local/wnaji.dev/wnaji.dev.crt","csr_pem":"###","identifier":"wnaji.dev","issuer":"local","metadata_path":"certificates/local/wnaji.dev/wnaji.dev.json","private_key_path":"certificates/local/wnaji.dev/wnaji.dev.key","renewal":false,"storage_path":"certificates/local/wnaji.dev"}}
{"level":"debug","ts":1735515098.7559724,"logger":"events","msg":"invoking subscribed handler","name":"cert_obtained","id":"ee815a16-1ec1-40b1-bce9-4ab3f9bfd451","origin":"tls","data":{"certificate_path":"certificates/local/wnaji.dev/wnaji.dev.crt","csr_pem":"###","identifier":"wnaji.dev","issuer":"local","metadata_path":"certificates/local/wnaji.dev/wnaji.dev.json","private_key_path":"certificates/local/wnaji.dev/wnaji.dev.key","renewal":false,"storage_path":"certificates/local/wnaji.dev"},"subscribed_to":"cert_obtained","handler":{"command":"sh","args":["-c","/scripts/cert_obtained.sh {event.data.certificate_path}"],"timeout":30000000000}}
{"level":"info","ts":1735515098.756312,"logger":"tls.obtain","msg":"releasing lock","identifier":"wnaji.dev"}
{"level":"warn","ts":1735515098.7569404,"logger":"tls","msg":"stapling OCSP","error":"no OCSP stapling for [wnaji.dev]: no OCSP server specified in certificate","identifiers":["wnaji.dev"]}
{"level":"debug","ts":1735515098.7571466,"logger":"tls.cache","msg":"added certificate to cache","subjects":["wnaji.dev"],"expiration":1735558299,"managed":true,"issuer_key":"local","hash":"f2f69f2a69dd9848eb2b0a53c53f98c14a4f7d78d8644f043bddacbba60c03c5","cache_size":5,"cache_capacity":10000}
{"level":"debug","ts":1735515098.7573564,"logger":"events","msg":"event","name":"cached_managed_cert","id":"068b68b0-63a5-4774-9407-4dd9f68ea4e7","origin":"tls","data":{"sans":["wnaji.dev"]}}
{"level":"info","ts":1735515098.759778,"logger":"tls.obtain","msg":"certificate obtained successfully","identifier":"pm.wnaji.dev","issuer":"local"}
{"level":"debug","ts":1735515098.7600188,"logger":"events","msg":"event","name":"cert_obtained","id":"8451f5c4-76b8-4c75-b095-ec6933c62a16","origin":"tls","data":{"certificate_path":"certificates/local/pm.wnaji.dev/pm.wnaji.dev.crt","csr_pem":"###","identifier":"pm.wnaji.dev","issuer":"local","metadata_path":"certificates/local/pm.wnaji.dev/pm.wnaji.dev.json","private_key_path":"certificates/local/pm.wnaji.dev/pm.wnaji.dev.key","renewal":false,"storage_path":"certificates/local/pm.wnaji.dev"}}
{"level":"debug","ts":1735515098.7601745,"logger":"events","msg":"invoking subscribed handler","name":"cert_obtained","id":"8451f5c4-76b8-4c75-b095-ec6933c62a16","origin":"tls","data":{"certificate_path":"certificates/local/pm.wnaji.dev/pm.wnaji.dev.crt","csr_pem":"###","identifier":"pm.wnaji.dev","issuer":"local","metadata_path":"certificates/local/pm.wnaji.dev/pm.wnaji.dev.json","private_key_path":"certificates/local/pm.wnaji.dev/pm.wnaji.dev.key","renewal":false,"storage_path":"certificates/local/pm.wnaji.dev"},"subscribed_to":"cert_obtained","handler":{"command":"sh","args":["-c","/scripts/cert_obtained.sh {event.data.certificate_path}"],"timeout":30000000000}}
{"level":"info","ts":1735515098.7602973,"logger":"tls.obtain","msg":"releasing lock","identifier":"pm.wnaji.dev"}
{"level":"warn","ts":1735515098.7608335,"logger":"tls","msg":"stapling OCSP","error":"no OCSP stapling for [pm.wnaji.dev]: no OCSP server specified in certificate","identifiers":["pm.wnaji.dev"]}
{"level":"debug","ts":1735515098.761933,"logger":"tls.cache","msg":"added certificate to cache","subjects":["pm.wnaji.dev"],"expiration":1735558299,"managed":true,"issuer_key":"local","hash":"1d915c9537d53be87f8e6d0ee7045fe8ac19c03e1f74a96c9cd0296558462457","cache_size":6,"cache_capacity":10000}
{"level":"debug","ts":1735515098.7620733,"logger":"events","msg":"event","name":"cached_managed_cert","id":"eb049d6c-d25e-4d3e-86b0-8af10b4cb72a","origin":"tls","data":{"sans":["pm.wnaji.dev"]}}
{"level":"info","ts":1735515098.762227,"logger":"tls.obtain","msg":"certificate obtained successfully","identifier":"wnaji.local","issuer":"local"}
{"level":"debug","ts":1735515098.7624822,"logger":"events","msg":"event","name":"cert_obtained","id":"73208226-52a4-4fa7-b599-3de069a6d425","origin":"tls","data":{"certificate_path":"certificates/local/wnaji.local/wnaji.local.crt","csr_pem":"###","identifier":"wnaji.local","issuer":"local","metadata_path":"certificates/local/wnaji.local/wnaji.local.json","private_key_path":"certificates/local/wnaji.local/wnaji.local.key","renewal":false,"storage_path":"certificates/local/wnaji.local"}}
{"level":"debug","ts":1735515098.7625778,"logger":"events","msg":"invoking subscribed handler","name":"cert_obtained","id":"73208226-52a4-4fa7-b599-3de069a6d425","origin":"tls","data":{"certificate_path":"certificates/local/wnaji.local/wnaji.local.crt","csr_pem":"###","identifier":"wnaji.local","issuer":"local","metadata_path":"certificates/local/wnaji.local/wnaji.local.json","private_key_path":"certificates/local/wnaji.local/wnaji.local.key","renewal":false,"storage_path":"certificates/local/wnaji.local"},"subscribed_to":"cert_obtained","handler":{"command":"sh","args":["-c","/scripts/cert_obtained.sh {event.data.certificate_path}"],"timeout":30000000000}}
{"level":"info","ts":1735515098.7626753,"logger":"tls.obtain","msg":"releasing lock","identifier":"wnaji.local"}
{"level":"warn","ts":1735515098.76309,"logger":"tls","msg":"stapling OCSP","error":"no OCSP stapling for [wnaji.local]: no OCSP server specified in certificate","identifiers":["wnaji.local"]}
{"level":"debug","ts":1735515098.7632318,"logger":"tls.cache","msg":"added certificate to cache","subjects":["wnaji.local"],"expiration":1735558299,"managed":true,"issuer_key":"local","hash":"0b9d078f048b54c9611b362caa90408412003a3e08e5fac128ea7caa586db879","cache_size":7,"cache_capacity":10000}
{"level":"debug","ts":1735515098.7633188,"logger":"events","msg":"event","name":"cached_managed_cert","id":"d5cd19ad-592f-488f-a717-cfa8e9f3505f","origin":"tls","data":{"sans":["wnaji.local"]}}
{"level":"info","ts":1735515098.7650416,"logger":"tls.obtain","msg":"certificate obtained successfully","identifier":"pmt.wnaji.dev","issuer":"local"}
{"level":"debug","ts":1735515098.765271,"logger":"events","msg":"event","name":"cert_obtained","id":"661771ba-692d-4d74-b48d-419d62cbf2a0","origin":"tls","data":{"certificate_path":"certificates/local/pmt.wnaji.dev/pmt.wnaji.dev.crt","csr_pem":"###","identifier":"pmt.wnaji.dev","issuer":"local","metadata_path":"certificates/local/pmt.wnaji.dev/pmt.wnaji.dev.json","private_key_path":"certificates/local/pmt.wnaji.dev/pmt.wnaji.dev.key","renewal":false,"storage_path":"certificates/local/pmt.wnaji.dev"}}
{"level":"debug","ts":1735515098.7653368,"logger":"events","msg":"invoking subscribed handler","name":"cert_obtained","id":"661771ba-692d-4d74-b48d-419d62cbf2a0","origin":"tls","data":{"certificate_path":"certificates/local/pmt.wnaji.dev/pmt.wnaji.dev.crt","csr_pem":"###","identifier":"pmt.wnaji.dev","issuer":"local","metadata_path":"certificates/local/pmt.wnaji.dev/pmt.wnaji.dev.json","private_key_path":"certificates/local/pmt.wnaji.dev/pmt.wnaji.dev.key","renewal":false,"storage_path":"certificates/local/pmt.wnaji.dev"},"subscribed_to":"cert_obtained","handler":{"command":"sh","args":["-c","/scripts/cert_obtained.sh {event.data.certificate_path}"],"timeout":30000000000}}
{"level":"info","ts":1735515098.7654548,"logger":"tls.obtain","msg":"releasing lock","identifier":"pmt.wnaji.dev"}
{"level":"warn","ts":1735515098.765855,"logger":"tls","msg":"stapling OCSP","error":"no OCSP stapling for [pmt.wnaji.dev]: no OCSP server specified in certificate","identifiers":["pmt.wnaji.dev"]}
{"level":"debug","ts":1735515098.765995,"logger":"tls.cache","msg":"added certificate to cache","subjects":["pmt.wnaji.dev"],"expiration":1735558299,"managed":true,"issuer_key":"local","hash":"6065d72c51f0165ff2ca1a279c854fdc3bcc96dfa7831bf4b8851eb5f276a6bd","cache_size":8,"cache_capacity":10000}
{"level":"debug","ts":1735515098.7661095,"logger":"events","msg":"event","name":"cached_managed_cert","id":"9eafad78-c9ce-4a7a-bd0e-09a61db937a8","origin":"tls","data":{"sans":["pmt.wnaji.dev"]}}
{"level":"info","ts":1735515103.862306,"msg":"using config from file","file":"/etc/caddy/Caddyfile"}
{"level":"info","ts":1735515103.8697217,"msg":"adapted config to JSON","adapter":"caddyfile"}
{"level":"warn","ts":1735515103.869749,"msg":"Caddyfile input is not formatted; run 'caddy fmt --overwrite' to fix inconsistencies","adapter":"caddyfile","file":"/etc/caddy/Caddyfile","line":1}
{"level":"info","ts":1735515103.8715878,"logger":"admin.api","msg":"received request","method":"POST","host":"localhost:2019","uri":"/load","remote_ip":"127.0.0.1","remote_port":"56564","headers":{"Accept-Encoding":["gzip"],"Cache-Control":["must-revalidate"],"Content-Length":["10752"],"Content-Type":["application/json"],"Origin":["http://localhost:2019"],"User-Agent":["Go-http-client/1.1"]}}
{"level":"info","ts":1735515103.8743622,"logger":"admin","msg":"admin endpoint started","address":"localhost:2019","enforce_origin":false,"origins":["//[::1]:2019","//127.0.0.1:2019","//localhost:2019"]}
{"level":"info","ts":1735515103.8755226,"logger":"http.auto_https","msg":"server is listening only on the HTTPS port but has no TLS connection policies; adding one to enable TLS","server_name":"srv0","https_port":443}
{"level":"info","ts":1735515103.8756557,"logger":"http.auto_https","msg":"enabling automatic HTTP->HTTPS redirects","server_name":"srv0"}
{"level":"warn","ts":1735515103.8756828,"logger":"http.auto_https","msg":"server is listening only on the HTTP port, so no automatic HTTPS will be applied to this server","server_name":"srv1","http_port":80}
{"level":"debug","ts":1735515103.8757353,"logger":"http.auto_https","msg":"adjusted config","tls":{"automation":{"policies":[{"subjects":["wnaji.local"]},{}]}},"http":{"servers":{"srv0":{"listen":[":443"],"routes":[{"handle":[{"handler":"subroute","routes":[{"handle":[{"REAL_HOST":"{http.request.host}","handler":"vars"},{"REAL_IP":"{http.request.remote.host}","handler":"vars"}]},{"handle":[{"REAL_HOST":"{http.request.header.X-Forwarded-Host}","handler":"vars"}],"match":[{"header":{"X-Forwarded-Host":["*"]}}]},{"handle":[{"REAL_IP":"{http.request.header.Cf-Connecting-Ip}","handler":"vars"}],"match":[{"header":{"Cf-Connecting-Ip":["*"]}}]},{"handle":[{"handler":"headers","response":{"set":{"X-Served-By":["Grafana"]}}},{"handler":"authentication","providers":{"authorizer":{"gatekeeper_name":"admins_policy","route_matcher":"*"}}}]},{"handle":[{"handler":"reverse_proxy","upstreams":[{"dial":"grafana:3000"}]}],"match":[{"header":{"Connection":["*Upgrade*"],"Upgrade":["websocket"]}}]},{"handle":[{"handler":"reverse_proxy","headers":{"request":{"set":{"Host":["{http.vars.REAL_HOST}"],"X-Real-Ip":["{http.vars.REAL_IP}"]}}},"upstreams":[{"dial":"grafana:3000"}]}]}]}],"terminal":true},{"handle":[{"handler":"subroute","routes":[{"handle":[{"handler":"subroute","routes":[{"handle":[{"handler":"subroute","routes":[{"handle":[{"handler":"authenticator","portal_name":"admin_portal","route_matcher":"*"}]}]}],"match":[{"path":["*"]}]}]}]}]}],"terminal":true},{"handle":[{"handler":"subroute","routes":[{"handle":[{"REAL_HOST":"{http.request.host}","handler":"vars"},{"REAL_IP":"{http.request.remote.host}","handler":"vars"}]},{"handle":[{"REAL_HOST":"{http.request.header.X-Forwarded-Host}","handler":"vars"}],"match":[{"header":{"X-Forwarded-Host":["*"]}}]},{"handle":[{"REAL_IP":"{http.request.header.Cf-Connecting-Ip}","handler":"vars"}],"match":[{"header":{"Cf-Connecting-Ip":["*"]}}]},{"handle":[{"handler":"headers","response":{"set":{"X-Served-By":["Loki"]}}},{"handler":"authentication","providers":{"authorizer":{"gatekeeper_name":"admins_policy","route_matcher":"*"}}}]},{"handle":[{"handler":"reverse_proxy","upstreams":[{"dial":"loki:3100"}]}],"match":[{"header":{"Connection":["*Upgrade*"],"Upgrade":["websocket"]}}]},{"handle":[{"handler":"reverse_proxy","headers":{"request":{"set":{"Host":["{http.vars.REAL_HOST}"],"X-Real-Ip":["{http.vars.REAL_IP}"]}}},"upstreams":[{"dial":"loki:3100"}]}]}]}],"terminal":true},{"handle":[{"handler":"subroute","routes":[{"handle":[{"REAL_HOST":"{http.request.host}","handler":"vars"},{"REAL_IP":"{http.request.remote.host}","handler":"vars"}]},{"handle":[{"REAL_HOST":"{http.request.header.X-Forwarded-Host}","handler":"vars"}],"match":[{"header":{"X-Forwarded-Host":["*"]}}]},{"handle":[{"REAL_IP":"{http.request.header.Cf-Connecting-Ip}","handler":"vars"}],"match":[{"header":{"Cf-Connecting-Ip":["*"]}}]},{"handle":[{"handler":"headers","response":{"set":{"X-Served-By":["promtail"]}}},{"handler":"authentication","providers":{"authorizer":{"gatekeeper_name":"admins_policy","route_matcher":"*"}}}]},{"handle":[{"handler":"reverse_proxy","upstreams":[{"dial":"promtail:9080"}]}],"match":[{"header":{"Connection":["*Upgrade*"],"Upgrade":["websocket"]}}]},{"handle":[{"handler":"reverse_proxy","headers":{"request":{"set":{"Host":["{http.vars.REAL_HOST}"],"X-Real-Ip":["{http.vars.REAL_IP}"]}}},"upstreams":[{"dial":"promtail:9080"}]}]}]}],"terminal":true},{"handle":[{"handler":"subroute","routes":[{"handle":[{"REAL_HOST":"{http.request.host}","handler":"vars"},{"REAL_IP":"{http.request.remote.host}","handler":"vars"}]},{"handle":[{"REAL_HOST":"{http.request.header.X-Forwarded-Host}","handler":"vars"}],"match":[{"header":{"X-Forwarded-Host":["*"]}}]},{"handle":[{"REAL_IP":"{http.request.header.Cf-Connecting-Ip}","handler":"vars"}],"match":[{"header":{"Cf-Connecting-Ip":["*"]}}]},{"handle":[{"handler":"headers","response":{"set":{"X-Served-By":["Alertmanager"]}}},{"handler":"authentication","providers":{"authorizer":{"gatekeeper_name":"admins_policy","route_matcher":"*"}}}]},{"handle":[{"handler":"reverse_proxy","upstreams":[{"dial":"alertmanager:9093"}]}],"match":[{"header":{"Connection":["*Upgrade*"],"Upgrade":["websocket"]}}]},{"handle":[{"handler":"reverse_proxy","headers":{"request":{"set":{"Host":["{http.vars.REAL_HOST}"],"X-Real-Ip":["{http.vars.REAL_IP}"]}}},"upstreams":[{"dial":"alertmanager:9093"}]}]}]}],"terminal":true},{"handle":[{"handler":"subroute","routes":[{"handle":[{"REAL_HOST":"{http.request.host}","handler":"vars"},{"REAL_IP":"{http.request.remote.host}","handler":"vars"}]},{"handle":[{"REAL_HOST":"{http.request.header.X-Forwarded-Host}","handler":"vars"}],"match":[{"header":{"X-Forwarded-Host":["*"]}}]},{"handle":[{"REAL_IP":"{http.request.header.Cf-Connecting-Ip}","handler":"vars"}],"match":[{"header":{"Cf-Connecting-Ip":["*"]}}]},{"handle":[{"handler":"headers","response":{"set":{"X-Served-By":["Prometheus"]}}},{"handler":"authentication","providers":{"authorizer":{"gatekeeper_name":"admins_policy","route_matcher":"*"}}}]},{"handle":[{"handler":"reverse_proxy","upstreams":[{"dial":"prometheus:9090"}]}],"match":[{"header":{"Connection":["*Upgrade*"],"Upgrade":["websocket"]}}]},{"handle":[{"handler":"reverse_proxy","headers":{"request":{"set":{"Host":["{http.vars.REAL_HOST}"],"X-Real-Ip":["{http.vars.REAL_IP}"]}}},"upstreams":[{"dial":"prometheus:9090"}]}]}]}],"terminal":true},{"handle":[{"handler":"subroute","routes":[{"handle":[{"REAL_HOST":"{http.request.host}","handler":"vars"},{"REAL_IP":"{http.request.remote.host}","handler":"vars"}]},{"handle":[{"REAL_HOST":"{http.request.header.X-Forwarded-Host}","handler":"vars"}],"match":[{"header":{"X-Forwarded-Host":["*"]}}]},{"handle":[{"REAL_IP":"{http.request.header.Cf-Connecting-Ip}","handler":"vars"}],"match":[{"header":{"Cf-Connecting-Ip":["*"]}}]},{"handle":[{"encodings":{"gzip":{}},"handler":"encode","prefer":["gzip"]}]},{"group":"group9","handle":[{"handler":"subroute","routes":[{"handle":[{"handler":"vars","root":"/usr/share/caddy"},{"handler":"headers","response":{"set":{"X-Served-By":["Caddy"]}}},{"handler":"file_server","hide":["/etc/caddy/sites-enabled/www.caddyfile","/etc/caddy/snippets/snippets_main.caddyfile"]}]}]},{"handler":"subroute","routes":[{"handle":[{"body":"Not found","handler":"static_response","status_code":404}]}]}]}]}],"terminal":true}],"tls_connection_policies":[{}],"automatic_https":{"prefer_wildcard":true},"trusted_proxies":{"ranges":["192.168.0.0/16","172.16.0.0/12","10.0.0.0/8","127.0.0.1/8","fd00::/8","::1"],"source":"static"},"client_ip_headers":["Cf-Connecting-Ip","X-Real-IP"],"logs":{"logger_names":{"admin.wnaji.dev":["log0"],"wnaji.dev":[""],"wnaji.local":[""]},"skip_hosts":["am.wnaji.dev","grafana.wnaji.dev","loki.wnaji.dev","pm.wnaji.dev","pmt.wnaji.dev"]}},"srv1":{"listen":[":80"],"routes":[{"handle":[{"handler":"subroute","routes":[{"handle":[{"abort":true,"handler":"static_response"}],"match":[{"not":[{"client_ip":{"ranges":["192.168.0.0/16","172.16.0.0/12","10.0.0.0/8","127.0.0.1/8","fd00::/8","::1"]}}]}]},{"handle":[{"handler":"metrics"}]}]}],"terminal":true},{},{}],"automatic_https":{"disable":true,"prefer_wildcard":true},"trusted_proxies":{"ranges":["192.168.0.0/16","172.16.0.0/12","10.0.0.0/8","127.0.0.1/8","fd00::/8","::1"],"source":"static"},"client_ip_headers":["Cf-Connecting-Ip","X-Real-IP"]}},"metrics":{}}}
{"level":"warn","ts":1735515103.9559255,"logger":"pki.ca.local","msg":"installing root certificate (you might be prompted for password)","path":"storage:pki/authorities/local/root.crt"}
{"level":"info","ts":1735515103.9563527,"msg":"warning: \"certutil\" is not available, install \"certutil\" with \"apt install libnss3-tools\" or \"yum install nss-tools\" and try again"}
{"level":"info","ts":1735515103.9563766,"msg":"define JAVA_HOME environment variable to use the Java trust"}
{"level":"info","ts":1735515104.0122855,"msg":"certificate installed properly in linux trusts"}
{"level":"debug","ts":1735515104.0124445,"logger":"security","msg":"started app instance","app":"security"}
{"level":"info","ts":1735515104.0124853,"logger":"http","msg":"enabling HTTP/3 listener","addr":":443"}
{"level":"debug","ts":1735515104.013814,"logger":"http","msg":"starting server loop","address":"[::]:443","tls":true,"http3":true}
{"level":"info","ts":1735515104.0138438,"logger":"http.log","msg":"server running","name":"srv0","protocols":["h1","h2","h3"]}
{"level":"warn","ts":1735515104.0139952,"logger":"http","msg":"HTTP/3 skipped because it requires TLS","network":"tcp","addr":":80"}
{"level":"debug","ts":1735515104.0140834,"logger":"http","msg":"starting server loop","address":"[::]:80","tls":false,"http3":false}
{"level":"warn","ts":1735515104.01412,"logger":"http","msg":"HTTP/2 skipped because it requires TLS","network":"tcp","addr":":80"}
{"level":"info","ts":1735515104.0141284,"logger":"http.log","msg":"server running","name":"srv1","protocols":["h1","h2","h3"]}
{"level":"info","ts":1735515104.0141377,"logger":"http","msg":"enabling automatic TLS certificate management","domains":["pm.wnaji.dev","wnaji.dev","wnaji.local","grafana.wnaji.dev","admin.wnaji.dev","loki.wnaji.dev","pmt.wnaji.dev","am.wnaji.dev"]}
{"level":"debug","ts":1735515104.0142326,"logger":"security","msg":"stopped app instance","app":"security"}
{"level":"info","ts":1735515104.0142446,"logger":"http","msg":"servers shutting down with eternal grace period"}
{"level":"info","ts":1735515104.0146883,"msg":"autosaved config (load with --resume flag)","file":"/config/caddy/autosave.json"}
{"level":"info","ts":1735515104.0149457,"logger":"admin.api","msg":"load complete"}
{"level":"error","ts":1735515104.014911,"logger":"events.handlers.exec","msg":"background command failed","error":"signal: killed"}
{"level":"info","ts":1735515104.0193648,"logger":"admin","msg":"stopped previous server","address":"localhost:2019"}
{"level":"debug","ts":1735515112.0454206,"logger":"events","msg":"event","name":"tls_get_certificate","id":"c8803fdd-e9b5-4585-8689-12c43d0ec380","origin":"tls","data":{"client_hello":{"CipherSuites":[4866,4867,4865,49196,49200,159,52393,52392,52394,49195,49199,158,49188,49192,107,49187,49191,103,49162,49172,57,49161,49171,51,157,156,61,60,53,47],"ServerName":"wnaji.local","SupportedCurves":[29,23,30,25,24,256,257,258,259,260],"SupportedPoints":"AAEC","SignatureSchemes":[1027,1283,1539,2055,2056,2074,2075,2076,2057,2058,2059,2052,2053,2054,1025,1281,1537,771,769,770,1026,1282,1538],"SupportedProtos":null,"SupportedVersions":[772,771],"RemoteAddr":{"IP":"185.106.*.*","Port":62822,"Zone":""},"LocalAddr":{"IP":"172.18.0.2","Port":443,"Zone":""}}}}
{"level":"debug","ts":1735515112.045781,"logger":"tls.handshake","msg":"choosing certificate","identifier":"wnaji.local","num_choices":1}
{"level":"debug","ts":1735515112.04579,"logger":"tls.handshake","msg":"default certificate selection results","identifier":"wnaji.local","subjects":["wnaji.local"],"managed":true,"issuer_key":"local","hash":"0b9d078f048b54c9611b362caa90408412003a3e08e5fac128ea7caa586db879"}
{"level":"debug","ts":1735515112.0458148,"logger":"tls.handshake","msg":"matched certificate in cache","remote_ip":"185.106.*.*","remote_port":"62822","subjects":["wnaji.local"],"managed":true,"expiration":1735558299,"hash":"0b9d078f048b54c9611b362caa90408412003a3e08e5fac128ea7caa586db879"}
{"level":"info","ts":1735515122.5332406,"msg":"shutting down apps, then terminating","signal":"SIGTERM"}
{"level":"warn","ts":1735515122.5332956,"msg":"exiting; byeee!! 👋","signal":"SIGTERM"}
{"level":"debug","ts":1735515122.5333157,"logger":"security","msg":"stopped app instance","app":"security"}
{"level":"info","ts":1735515122.5333645,"logger":"http","msg":"servers shutting down with eternal grace period"}
{"level":"info","ts":1735515122.5339742,"logger":"admin","msg":"stopped previous server","address":"localhost:2019"}
{"level":"info","ts":1735515122.5339904,"msg":"shutdown complete","signal":"SIGTERM","exit_code":0}
3. Caddy version:
v2.9.0-beta.3 h1:tlqfbJMRNY6vnWwaQrnWrgS+wkDXr9GIFUD/P+HY9vA=
4. How I installed and ran Caddy:
FROM caddy:2.9-builder-alpine AS builder
RUN xcaddy build \
--with github.com/greenpau/caddy-security@v1.1.29 \
--with github.com/caddyserver/cache-handler@v0.15.0 \
--with github.com/darkweak/storages/otter/caddy \
--with github.com/porech/caddy-maxmind-geolocation \
--with github.com/mholt/caddy-events-exec
FROM caddy:2.9-alpine
COPY --from=builder /usr/bin/caddy /usr/bin/caddy
COPY --chown=root:root ./ca/ /data/myca/
COPY --chown=root:root --chmod=700 ./scripts/ /scripts/
RUN chmod +x -R /scripts
EXPOSE 80
EXPOSE 443
CMD ["caddy", "run", "--config", "/etc/caddy/Caddyfile"]
a. System environment:
Docker version 27.4.0, build bde2b89
b. Command:
#!/bin/sh
CERT_PATH="/data/caddy/$1"
FULLCHAIN_PATH="/data/myca/fullchain.crt"
LAST_CHANGED_FP="/tmp/last_changed"
# Ensure required files exist
if [ ! -f "$CERT_PATH" ]; then
echo "Error: Certificate file $CERT_PATH does not exist."
exit 1
fi
if [ ! -f "$FULLCHAIN_PATH" ]; then
echo "Error: Fullchain file $FULLCHAIN_PATH does not exist."
exit 1
fi
# Backup current cert
cp "$CERT_PATH" "$CERT_PATH.bak" || {
echo "Error: Failed to back up $CERT_PATH."
exit 1
}
# Make fullchain
cat "$CERT_PATH.bak" "$FULLCHAIN_PATH" > "$CERT_PATH" || {
echo "Error: Failed to create fullchain at $CERT_PATH."
exit 1
}
# Record last changed file
echo "$CERT_PATH" > "$LAST_CHANGED_FP"
# Wait for 5 seconds to debounce reloads
sleep 5
# Check if this is still the last changed file
LAST_CHANGED=$(cat "$LAST_CHANGED_FP")
if [ "$LAST_CHANGED" = "$CERT_PATH" ]; then
# Reload Caddy
caddy reload --config "$CADDY_CONFIG" --force || {
echo "Error: Failed to reload Caddy."
exit 1
}
# Clean up
rm "$LAST_CHANGED_FP"
fi
c. Service/unit/compose file:
caddy:
networks:
- caddy
restart: unless-stopped
build: ./caddy
container_name: caddy
ports:
- 80:80
- 443:443
# - 2019:2019
volumes:
- ./caddy/Caddyfile:/etc/caddy/Caddyfile
- ./caddy/sites:/etc/caddy/sites-enabled
- ./caddy/snippets:/etc/caddy/snippets
- ./caddy/scripts:/scripts
# - caddy_data:/data
- caddy_logs:/logs
- geo_db:/geodb
environment:
- CADDY_CONFIG=/etc/caddy/Caddyfile
- CADDY_GLOBAL_CONFIG=${CADDY_GLOBAL_CONFIG}
- CADDY_JWT_SHARED_KEY=${CADDY_JWT_SHARED_KEY}
- CADDY_AUTH_USER=${CADDY_AUTH_USER}
- CADDY_AUTH_EMAIL=${CADDY_AUTH_EMAIL}
- CADDY_AUTH_PASSWORD=${CADDY_AUTH_PASSWORD}
d. My complete Caddy config:
My caddy file is broken up to multiple snippets and segments which is working fine. I will show the snippet related to PKI below. Otherwise everything is running ok
{
{$CADDY_GLOBAL_CONFIG}
# Enable metrics
metrics
servers {
trusted_proxies static private_ranges
client_ip_headers Cf-Connecting-Ip X-Real-IP
}
# Import other globals
import /etc/caddy/snippets/globals_*.caddyfile
}
globals_pki.caddyfile
pki {
ca {
intermediate {
cert /data/myca/caddy-root.crt
key /data/myca/caddy-root.key
}
}
}
# Enable local internal certificates
local_certs
auto_https prefer_wildcard
events {
on cert_obtained exec sh -c "/scripts/cert_obtained.sh {event.data.certificate_path}"
}