1. The problem I’m having:
I’m testing Caddy to generate certificates for multiple domains.
I have Caddy deployed in HA, by using 2 docker containers and having TLS certificates stored in Redis, using github.com/pberkel/caddy-storage-redis
.
I also run a http-challenge API endpoint to check a couple of things before replying 200 OK to Caddy to issue the certificate.
I tested the following:
$ curl https://localhost -k
Ok
# the above works as expected
$ curl https://localhost -k
Ok
# the above goes to the second Caddy server which contacts the http challenge server.
I was expecting the second Caddy server to check Redis and serve the request, instead of checking with the http challenge server.
What’s the flow Caddy’s doing? How can I avoid this behavior?
2. Error messages and/or full log output:
httpchallenge-1 | [2024-05-09 13:10:12] INFO Received TLS certificate request for localhost
caddy-1 | {"level":"info","ts":1715260212.5371542,"logger":"tls.on_demand","msg":"obtaining new certificate","remote_ip":"192.168.65.1","remote_port":"29736","server_name":"localhost"}
caddy-1 | {"level":"info","ts":1715260212.5388484,"logger":"tls.obtain","msg":"acquiring lock","identifier":"localhost"}
caddy-1 | {"level":"info","ts":1715260212.539165,"logger":"tls.obtain","msg":"lock acquired","identifier":"localhost"}
caddy-1 | {"level":"info","ts":1715260212.5395622,"logger":"tls.obtain","msg":"obtaining certificate","identifier":"localhost"}
caddy-1 | {"level":"info","ts":1715260212.5423813,"logger":"tls.obtain","msg":"certificate obtained successfully","identifier":"localhost"}
caddy-1 | {"level":"info","ts":1715260212.5424857,"logger":"tls.obtain","msg":"releasing lock","identifier":"localhost"}
caddy-1 | {"level":"warn","ts":1715260212.5436988,"logger":"tls","msg":"stapling OCSP","error":"no OCSP stapling for [localhost]: no OCSP server specified in certificate","identifiers":["localhost"]}
caddy-1 | {"level":"info","ts":1715260212.596209,"logger":"http.log.access.log0","msg":"handled request","request":{"remote_ip":"192.168.65.1","remote_port":"29736","client_ip":"192.168.65.1","proto":"HTTP/2.0","method":"GET","host":"localhost","uri":"/","headers":{"User-Agent":["curl/8.4.0"],"Accept":["*/*"]},"tls":{"resumed":false,"version":772,"cipher_suite":4867,"proto":"h2","server_name":"localhost"}},"bytes_read":0,"user_id":"","duration":0.000130959,"size":7,"status":200,"resp_headers":{"Server":["Caddy"],"Alt-Svc":["h3=\":443\"; ma=2592000"],"Content-Type":["text/plain; charset=utf-8"]}}
caddy-1 | {"level":"info","ts":1715260215.5124705,"logger":"http.log.access.log0","msg":"handled request","request":{"remote_ip":"192.168.65.1","remote_port":"29737","client_ip":"192.168.65.1","proto":"HTTP/2.0","method":"GET","host":"localhost","uri":"/","headers":{"User-Agent":["curl/8.4.0"],"Accept":["*/*"]},"tls":{"resumed":false,"version":772,"cipher_suite":4867,"proto":"h2","server_name":"localhost"}},"bytes_read":0,"user_id":"","duration":0.000089417,"size":7,"status":200,"resp_headers":{"Server":["Caddy"],"Alt-Svc":["h3=\":443\"; ma=2592000"],"Content-Type":["text/plain; charset=utf-8"]}}
caddy-1 | {"level":"info","ts":1715260216.3046443,"logger":"http.log.access.log0","msg":"handled request","request":{"remote_ip":"192.168.65.1","remote_port":"29738","client_ip":"192.168.65.1","proto":"HTTP/2.0","method":"GET","host":"localhost","uri":"/","headers":{"User-Agent":["curl/8.4.0"],"Accept":["*/*"]},"tls":{"resumed":false,"version":772,"cipher_suite":4867,"proto":"h2","server_name":"localhost"}},"bytes_read":0,"user_id":"","duration":0.00007275,"size":7,"status":200,"resp_headers":{"Server":["Caddy"],"Alt-Svc":["h3=\":443\"; ma=2592000"],"Content-Type":["text/plain; charset=utf-8"]}}
caddy-1 | {"level":"info","ts":1715260216.9100282,"logger":"http.log.access.log0","msg":"handled request","request":{"remote_ip":"192.168.65.1","remote_port":"29739","client_ip":"192.168.65.1","proto":"HTTP/2.0","method":"GET","host":"localhost","uri":"/","headers":{"User-Agent":["curl/8.4.0"],"Accept":["*/*"]},"tls":{"resumed":false,"version":772,"cipher_suite":4867,"proto":"h2","server_name":"localhost"}},"bytes_read":0,"user_id":"","duration":0.000097042,"size":7,"status":200,"resp_headers":{"Server":["Caddy"],"Alt-Svc":["h3=\":443\"; ma=2592000"],"Content-Type":["text/plain; charset=utf-8"]}}
caddy-1 | {"level":"info","ts":1715260217.5609527,"logger":"http.log.access.log0","msg":"handled request","request":{"remote_ip":"192.168.65.1","remote_port":"29740","client_ip":"192.168.65.1","proto":"HTTP/2.0","method":"GET","host":"localhost","uri":"/","headers":{"User-Agent":["curl/8.4.0"],"Accept":["*/*"]},"tls":{"resumed":false,"version":772,"cipher_suite":4867,"proto":"h2","server_name":"localhost"}},"bytes_read":0,"user_id":"","duration":0.000065416,"size":7,"status":200,"resp_headers":{"Server":["Caddy"],"Alt-Svc":["h3=\":443\"; ma=2592000"],"Content-Type":["text/plain; charset=utf-8"]}}
caddy-1 | {"level":"info","ts":1715260218.129712,"logger":"http.log.access.log0","msg":"handled request","request":{"remote_ip":"192.168.65.1","remote_port":"29741","client_ip":"192.168.65.1","proto":"HTTP/2.0","method":"GET","host":"localhost","uri":"/","headers":{"User-Agent":["curl/8.4.0"],"Accept":["*/*"]},"tls":{"resumed":false,"version":772,"cipher_suite":4867,"proto":"h2","server_name":"localhost"}},"bytes_read":0,"user_id":"","duration":0.000091667,"size":7,"status":200,"resp_headers":{"Server":["Caddy"],"Alt-Svc":["h3=\":443\"; ma=2592000"],"Content-Type":["text/plain; charset=utf-8"]}}
caddy-1 | {"level":"info","ts":1715260218.8676667,"logger":"http.log.access.log0","msg":"handled request","request":{"remote_ip":"192.168.65.1","remote_port":"29742","client_ip":"192.168.65.1","proto":"HTTP/2.0","method":"GET","host":"localhost","uri":"/","headers":{"Accept":["*/*"],"User-Agent":["curl/8.4.0"]},"tls":{"resumed":false,"version":772,"cipher_suite":4867,"proto":"h2","server_name":"localhost"}},"bytes_read":0,"user_id":"","duration":0.000124875,"size":7,"status":200,"resp_headers":{"Alt-Svc":["h3=\":443\"; ma=2592000"],"Content-Type":["text/plain; charset=utf-8"],"Server":["Caddy"]}}
caddy-1 | {"level":"info","ts":1715260219.5251272,"logger":"http.log.access.log0","msg":"handled request","request":{"remote_ip":"192.168.65.1","remote_port":"29743","client_ip":"192.168.65.1","proto":"HTTP/2.0","method":"GET","host":"localhost","uri":"/","headers":{"User-Agent":["curl/8.4.0"],"Accept":["*/*"]},"tls":{"resumed":false,"version":772,"cipher_suite":4867,"proto":"h2","server_name":"localhost"}},"bytes_read":0,"user_id":"","duration":0.000068083,"size":7,"status":200,"resp_headers":{"Content-Type":["text/plain; charset=utf-8"],"Server":["Caddy"],"Alt-Svc":["h3=\":443\"; ma=2592000"]}}
httpchallenge-2 | [2024-05-09 13:10:20] INFO Received TLS certificate request for localhost
caddy-2 | {"level":"warn","ts":1715260220.1596973,"logger":"tls","msg":"stapling OCSP","error":"no OCSP stapling for [localhost]: no OCSP server specified in certificate","identifiers":["localhost"]}
caddy-2 | {"level":"info","ts":1715260220.165014,"logger":"http.log.access.log0","msg":"handled request","request":{"remote_ip":"192.168.65.1","remote_port":"29750","client_ip":"192.168.65.1","proto":"HTTP/2.0","method":"GET","host":"localhost","uri":"/","headers":{"User-Agent":["curl/8.4.0"],"Accept":["*/*"]},"tls":{"resumed":false,"version":772,"cipher_suite":4867,"proto":"h2","server_name":"localhost"}},"bytes_read":0,"user_id":"","duration":0.000032833,"size":6,"status":200,"resp_headers":{"Content-Type":["text/plain; charset=utf-8"],"Server":["Caddy"],"Alt-Svc":["h3=\":443\"; ma=2592000"]}}
3. Caddy version:
caddy:2.7.6-builder
docker container.
4. How I installed and ran Caddy:
Docker containers.
a. System environment:
Docker
b. Command:
docker-compose up
c. Service/unit/compose file:
services:
caddy-1:
build:
dockerfile: ./caddy/caddy.Dockerfile
container_name: caddy-1
environment:
- REDIS_HOST=redis
- REDIS_PORT=6379
- REDIS_USERNAME=default
- REDIS_PASSWORD=redis
- HTTP_CHALLENGE_SERVER=http-challenge-1
volumes:
- ./caddy/Caddyfile:/etc/caddy/Caddyfile
caddy-2:
build:
dockerfile: ./caddy/caddy.Dockerfile
container_name: caddy-2
environment:
- REDIS_HOST=redis
- REDIS_PORT=6379
- REDIS_USERNAME=default
- REDIS_PASSWORD=redis
- HTTP_CHALLENGE_SERVER=http-challenge-2
volumes:
- ./caddy/Caddyfile:/etc/caddy/Caddyfile:ro
http-challenge-1:
build:
dockerfile: ./Dockerfile
container_name: http-challenge-1
http-challenge-2:
build:
dockerfile: ./Dockerfile
container_name: http-challenge-2
redis:
image: redis:7
container_name: redis
restart: always
command: sh -c redis-server --requirepass "$${REDIS_PASSWORD}"
ports:
- 6379:6379
environment:
- REDIS_PASSWORD=redis
d. My complete Caddy config:
{
on_demand_tls {
ask http://{$HTTP_CHALLENGE_SERVER}/ask
}
storage redis {
host {$REDIS_HOST}
port {$REDIS_PORT}
username {$REDIS_USERNAME}
password {$REDIS_PASSWORD}
}
}
https://* {
tls internal {
on_demand
}
handle /* {
respond Ok
}
respond 404
}