1. Output of caddy version:
daniel@dilithium:~/caddy$ sudo docker exec caddy caddy version
v2.6.2 h1:wKoFIxpmOJLGl3QXoo6PNbYvGW4xLEgo32GPBEjWL8o=
2. How I run Caddy:
Caddy runs in Docker on a macvlan network and is a reverse proxy for a single domain (at the moment), which is my Emby media server.
a. System environment:
Server: Bare Metal Ubuntu 20.04
Docker Version: 20.10.22
daniel@dilithium:~$ uname -a
Linux dilithium 5.4.0-135-generic #152-Ubuntu SMP Wed Nov 23 20:19:22 UTC 2022 x86_64 x86_64 x86_64 GNU/Linux
b. Command:
sudo docker compose start caddy
c. Service/unit/compose file:
services:
caddy:
build: ./dockerfile-caddy
container_name: caddy
hostname: caddy
restart: unless-stopped
ports:
- "80:80"
- "443:443"
- "443:443/udp"
environment:
- MY_DOMAIN
- CLOUDFLARE_API_TOKEN
volumes:
- ./Caddyfile:/etc/caddy/Caddyfile
- ./data:/data
- ./config:/config
networks:
default:
name: phynet6
external: true
docker-compose.yml
FROM caddy:2.6.2-builder AS builder
RUN xcaddy build \
--with github.com/caddy-dns/cloudflare
FROM caddy:2.6.2
COPY --from=builder /usr/bin/caddy /usr/bin/caddy
### I don't think this does anything, I just used it for testing
EXPOSE 80/tcp
EXPOSE 443/tcp
EXPOSE 443/udp
d. My complete Caddy config:
This is my actual Caddyfile with nothing changed
{
acme_dns cloudflare {$CLOUDFLARE_API_TOKEN}
}
emby.{$MY_DOMAIN} {
reverse_proxy https://[2600:1700:ada8:750f::3002]:8920 {
transport http {
tls_server_name emby.danielmarks.dev
tls_insecure_skip_verify
versions 2
}
}
}
3. The problem I’m having:
Caddy is not proxying over HTTP/3, and I’m not getting a connection via QUIC from either a browser or curl. This used to work back when we had to use the experimental_http3 flag, but now it won’t even negotiate http3.
http/2 and http/1.1 works as expected
4. Error messages and/or full log output:
Logs when server is initialized
{"level":"info","ts":1671966298.820925,"msg":"using provided configuration","config_file":"/etc/caddy/Caddyfile","config_adapter":"caddyfile"}
{"level":"warn","ts":1671966298.8223689,"msg":"Caddyfile input is not formatted; run the 'caddy fmt' command to fix inconsistencies","adapter":"caddyfile","file":"/etc/caddy/Caddyfile","line":2}
{"level":"info","ts":1671966298.8233094,"logger":"admin","msg":"admin endpoint started","address":"localhost:2019","enforce_origin":false,"origins":["//localhost:2019","//[::1]:2019","//127.0.0.1:2019"]}
{"level":"info","ts":1671966298.8235204,"logger":"http","msg":"server is listening only on the HTTPS port but has no TLS connection policies; adding one to enable TLS","server_name":"srv0","https_port":443}
{"level":"info","ts":1671966298.8235352,"logger":"http","msg":"enabling automatic HTTP->HTTPS redirects","server_name":"srv0"}
{"level":"info","ts":1671966298.823603,"logger":"tls.cache.maintenance","msg":"started background certificate maintenance","cache":"0xc000492850"}
{"level":"info","ts":1671966298.8238966,"logger":"tls","msg":"cleaning storage unit","description":"FileStorage:/data/caddy"}
{"level":"info","ts":1671966298.8239021,"logger":"http","msg":"enabling HTTP/3 listener","addr":":443"}
{"level":"debug","ts":1671966298.8240638,"logger":"http","msg":"starting server loop","address":"[::]:443","tls":true,"http3":true}
{"level":"info","ts":1671966298.8240795,"logger":"http.log","msg":"server running","name":"srv0","protocols":["h1","h2","h3"]}
{"level":"debug","ts":1671966298.8241053,"logger":"http","msg":"starting server loop","address":"[::]:80","tls":false,"http3":false}
{"level":"info","ts":1671966298.8241127,"logger":"http.log","msg":"server running","name":"remaining_auto_https_redirects","protocols":["h1","h2","h3"]}
{"level":"info","ts":1671966298.8241162,"logger":"http","msg":"enabling automatic TLS certificate management","domains":["emby.danielmarks.dev"]}
{"level":"debug","ts":1671966298.8244905,"logger":"tls","msg":"loading managed certificate","domain":"emby.danielmarks.dev","expiration":1678652887,"issuer_key":"acme-v02.api.letsencrypt.org-directory","storage":"FileStorage:/data/caddy"}
{"level":"info","ts":1671966298.824511,"logger":"tls","msg":"finished cleaning storage units"}
{"level":"debug","ts":1671966298.8247864,"logger":"tls.cache","msg":"added certificate to cache","subjects":["emby.danielmarks.dev"],"expiration":1678652887,"managed":true,"issuer_key":"acme-v02.api.letsencrypt.org-directory","hash":"dccb5f3436632db870b67c49d705580dd7edeb8683b97529e46231bc7d205d26","cache_size":1,"cache_capacity":10000}
{"level":"debug","ts":1671966298.8248105,"logger":"events","msg":"event","name":"cached_managed_cert","id":"0d168666-f017-4944-812a-ec0c2be8b065","origin":"tls","data":{"sans":["emby.danielmarks.dev"]}}
{"level":"info","ts":1671966298.8249636,"msg":"autosaved config (load with --resume flag)","file":"/config/caddy/autosave.json"}
{"level":"info","ts":1671966298.824974,"msg":"serving initial configuration"}
Log when I query the server with HTTP/3 via curl
{"level":"debug","ts":1671966623.1476204,"logger":"events","msg":"event","name":"tls_get_certificate","id":"37162c87-98ab-4ca5-85c3-96667f789ae5","origin":"tls","data":{"client_hello":{"CipherSuites":[4865,4866,4867],"ServerName":"emby.danielmarks.dev","SupportedCurves":[29,23,24],"SupportedPoints":null,"SignatureSchemes":[1027,2052,1025,1283,2053,1281,2054,1537,513],"SupportedProtos":["h3","h3-29","h3-28","h3-27"],"SupportedVersions":[772],"Conn":{}}}}
{"level":"debug","ts":1671966623.1477995,"logger":"tls.handshake","msg":"choosing certificate","identifier":"emby.danielmarks.dev","num_choices":1}
{"level":"debug","ts":1671966623.1478229,"logger":"tls.handshake","msg":"default certificate selection results","identifier":"emby.danielmarks.dev","subjects":["emby.danielmarks.dev"],"managed":true,"issuer_key":"acme-v02.api.letsencrypt.org-directory","hash":"dccb5f3436632db870b67c49d705580dd7edeb8683b97529e46231bc7d205d26"}
{"level":"debug","ts":1671966623.1478362,"logger":"tls.handshake","msg":"matched certificate in cache","remote_ip":"2600:1700:ada8:750f:419e:fef9:b6ae:caab","remote_port":"50913","subjects":["emby.danielmarks.dev"],"managed":true,"expiration":1678652887,"hash":"dccb5f3436632db870b67c49d705580dd7edeb8683b97529e46231bc7d205d26"}
5. What I already tried:
Here’s a curl output (patched with quiche):
danielmarks@Daniels-MBP ~ % curl --http3 https://emby.danielmarks.dev:443 -v
* Trying [2600:1700:ada8:750f::3001]:443...
* CAfile: /etc/ssl/cert.pem
* CApath: none
* Connect socket 5 over QUIC to 2600:1700:ada8:750f::3001:443
* Sent QUIC client Initial, ALPN: h3,h3-29,h3-28,h3-27
* Connection timeout after 300000 ms
* Closing connection 0
curl: (28) Connection timeout after 300000 ms
I’ve run pcaps and found that the caddy server is not attempting to respond at all.
aforementioned pcap: http://global.danielmarks.dev/ds/quic-pcap.pcapng
Perhaps Caddy doesn’t proxy HTTP/3 when the origin is HTTP/2?
