1. The problem I’m having:
Hello, please help. I can’t set up a secure http2 reverse proxy from a client via a cname domain located on the cdn in front of the caddy, but when connecting directly to the origin domain everything works fine, it feels like some response header is missing, I’ve been racking my brains about this for a long time.
2. Error messages and/or full log output:
This is a piece of log with a successful connection to origin domain on caddy server:
{"level":"debug","ts":1707068297.3494523,"logger":"events","msg":"event","name":"tls_get_certificate","id":"ed531aee-4cb1-40a5-b88f-ca4bc9ca1c50","origin":"tls","data":{"client_hello":{"CipherSuites":[23130,4865,4866,4867,49195,49199,49196,49200,52393,52392,49171,49172,156,157,47,53],"ServerName":"gkm.pw","SupportedCurves":[10794,29,23,24],"SupportedPoints":"AA==","SignatureSchemes":[1027,2052,1025,1283,2053,1281,2054,1537],"SupportedProtos":["h2","http/1.1"],"SupportedVersions":[10794,772,771],"RemoteAddr":{"IP":"193.200.74.177","Port":4840,"Zone":""},"LocalAddr":{"IP":"31.129.111.230","Port":443,"Zone":""}}}}
{"level":"debug","ts":1707068297.350322,"logger":"tls.handshake","msg":"choosing certificate","identifier":"gkm.pw","num_choices":1}
{"level":"debug","ts":1707068297.3505447,"logger":"tls.handshake","msg":"custom certificate selection results","identifier":"gkm.pw","subjects":["gkm.pw"],"managed":false,"issuer_key":"","hash":"e31c18709c42f5db2eae3032047b22e34cf6f3c1bafb24e4d80dc2deba9d5e01"}
{"level":"debug","ts":1707068297.3506336,"logger":"tls.handshake","msg":"matched certificate in cache","remote_ip":"193.200.74.177","remote_port":"4840","subjects":["gkm.pw"],"managed":false,"expiration":1713553201,"hash":"e31c18709c42f5db2eae3032047b22e34cf6f3c1bafb24e4d80dc2deba9d5e01"}
{"level":"debug","ts":1707068297.5013633,"logger":"http.handlers.reverse_proxy","msg":"selected upstream","dial":"127.0.0.1:15450","total_upstreams":1}
{"level":"debug","ts":1707068297.5038974,"logger":"http.handlers.reverse_proxy","msg":"upstream roundtrip","upstream":"127.0.0.1:15450","duration":0.002250472,"request":{"remote_ip":"193.200.74.177","remote_port":"4840","client_ip":"193.200.74.177","proto":"HTTP/2.0","method":"PUT","host":"gkm.pw","uri":"/akkl190219","headers":{"User-Agent":["Go-http-client/2.0"],"Accept-Encoding":["identity"],"X-Forwarded-For":["193.200.74.177"],"X-Forwarded-Proto":["https"],"X-Forwarded-Host":["gkm.pw"]},"tls":{"resumed":false,"version":772,"cipher_suite":4865,"proto":"h2","server_name":"gkm.pw"}},"headers":{"Cache-Control":["no-store"],"Date":["Sun, 04 Feb 2024 17:38:17 GMT"]},"status":200}
{"level":"error","ts":1707068297.9200675,"logger":"http.handlers.reverse_proxy","msg":"reading from backend","error":"client disconnected"}
{"level":"error","ts":1707068297.920316,"logger":"http.handlers.reverse_proxy","msg":"aborting with incomplete response","upstream":"127.0.0.1:15450","duration":0.002250472,"request":{"remote_ip":"193.200.74.177","remote_port":"4840","client_ip":"193.200.74.177","proto":"HTTP/2.0","method":"PUT","host":"gkm.pw","uri":"/akkl190219","headers":{"User-Agent":["Go-http-client/2.0"],"Accept-Encoding":["identity"],"X-Forwarded-For":["193.200.74.177"],"X-Forwarded-Proto":["https"],"X-Forwarded-Host":["gkm.pw"]},"tls":{"resumed":false,"version":772,"cipher_suite":4865,"proto":"h2","server_name":"gkm.pw"}},"error":"reading: client disconnected"}
{"level":"debug","ts":1707068300.3483403,"logger":"events","msg":"event","name":"tls_get_certificate","id":"817841ea-f05e-43eb-b21c-49ddb4e71c1d","origin":"tls","data":{"client_hello":{"CipherSuites":[56026,4865,4866,4867,49195,49199,49196,49200,52393,52392,49171,49172,156,157,47,53],"ServerName":"gkm.pw","SupportedCurves":[23130,29,23,24],"SupportedPoints":"AA==","SignatureSchemes":[1027,2052,1025,1283,2053,1281,2054,1537],"SupportedProtos":["h2","http/1.1"],"SupportedVersions":[64250,772,771],"RemoteAddr":{"IP":"193.200.74.177","Port":54360,"Zone":""},"LocalAddr":{"IP":"31.129.111.230","Port":443,"Zone":""}}}}
{"level":"debug","ts":1707068300.3491151,"logger":"tls.handshake","msg":"choosing certificate","identifier":"gkm.pw","num_choices":1}
{"level":"debug","ts":1707068300.3492475,"logger":"tls.handshake","msg":"custom certificate selection results","identifier":"gkm.pw","subjects":["gkm.pw"],"managed":false,"issuer_key":"","hash":"e31c18709c42f5db2eae3032047b22e34cf6f3c1bafb24e4d80dc2deba9d5e01"}
{"level":"debug","ts":1707068300.349319,"logger":"tls.handshake","msg":"matched certificate in cache","remote_ip":"193.200.74.177","remote_port":"54360","subjects":["gkm.pw"],"managed":false,"expiration":1713553201,"hash":"e31c18709c42f5db2eae3032047b22e34cf6f3c1bafb24e4d80dc2deba9d5e01"}
{"level":"debug","ts":1707068300.4986858,"logger":"http.handlers.reverse_proxy","msg":"selected upstream","dial":"127.0.0.1:15450","total_upstreams":1}
{"level":"debug","ts":1707068300.500581,"logger":"http.handlers.reverse_proxy","msg":"upstream roundtrip","upstream":"127.0.0.1:15450","duration":0.001664381,"request":{"remote_ip":"193.200.74.177","remote_port":"54360","client_ip":"193.200.74.177","proto":"HTTP/2.0","method":"PUT","host":"gkm.pw","uri":"/akkl190219","headers":{"Accept-Encoding":["identity"],"User-Agent":["Go-http-client/2.0"],"X-Forwarded-For":["193.200.74.177"],"X-Forwarded-Proto":["https"],"X-Forwarded-Host":["gkm.pw"]},"tls":{"resumed":false,"version":772,"cipher_suite":4865,"proto":"h2","server_name":"gkm.pw"}},"headers":{"Cache-Control":["no-store"],"Date":["Sun, 04 Feb 2024 17:38:20 GMT"]},"status":200}
{"level":"error","ts":1707068300.6445847,"logger":"http.handlers.reverse_proxy","msg":"reading from backend","error":"stream error: stream ID 1; CANCEL"}
{"level":"error","ts":1707068300.6448092,"logger":"http.handlers.reverse_proxy","msg":"aborting with incomplete response","upstream":"127.0.0.1:15450","duration":0.001664381,"request":{"remote_ip":"193.200.74.177","remote_port":"54360","client_ip":"193.200.74.177","proto":"HTTP/2.0","method":"PUT","host":"gkm.pw","uri":"/akkl190219","headers":{"Accept-Encoding":["identity"],"User-Agent":["Go-http-client/2.0"],"X-Forwarded-For":["193.200.74.177"],"X-Forwarded-Proto":["https"],"X-Forwarded-Host":["gkm.pw"]},"tls":{"resumed":false,"version":772,"cipher_suite":4865,"proto":"h2","server_name":"gkm.pw"}},"error":"reading: stream error: stream ID 1; CANCEL"}
{"level":"debug","ts":1707068301.490227,"logger":"http.handlers.reverse_proxy","msg":"selected upstream","dial":"127.0.0.1:15450","total_upstreams":1}
{"level":"debug","ts":1707068301.491789,"logger":"http.handlers.reverse_proxy","msg":"upstream roundtrip","upstream":"127.0.0.1:15450","duration":0.00119128,"request":{"remote_ip":"193.200.74.177","remote_port":"54360","client_ip":"193.200.74.177","proto":"HTTP/2.0","method":"PUT","host":"gkm.pw","uri":"/akkl190219","headers":{"Accept-Encoding":["identity"],"User-Agent":["Go-http-client/2.0"],"X-Forwarded-For":["193.200.74.177"],"X-Forwarded-Proto":["https"],"X-Forwarded-Host":["gkm.pw"]},"tls":{"resumed":false,"version":772,"cipher_suite":4865,"proto":"h2","server_name":"gkm.pw"}},"headers":{"Cache-Control":["no-store"],"Date":["Sun, 04 Feb 2024 17:38:21 GMT"]},"status":200}
{"level":"debug","ts":1707068307.1014218,"logger":"http.stdlib","msg":"http: TLS handshake error from 163.181.26.161:60012: EOF"}
{"level":"error","ts":1707068310.0027092,"logger":"http.handlers.reverse_proxy","msg":"aborting with incomplete response","upstream":"127.0.0.1:15450","duration":0.00119128,"request":{"remote_ip":"193.200.74.177","remote_port":"54360","client_ip":"193.200.74.177","proto":"HTTP/2.0","method":"PUT","host":"gkm.pw","uri":"/akkl190219","headers":{"Accept-Encoding":["identity"],"User-Agent":["Go-http-client/2.0"],"X-Forwarded-For":["193.200.74.177"],"X-Forwarded-Proto":["https"],"X-Forwarded-Host":["gkm.pw"]},"tls":{"resumed":false,"version":772,"cipher_suite":4865,"proto":"h2","server_name":"gkm.pw"}},"error":"writing: http2: stream closed"}
.
And this is a log with an unsuccessful connection to the origin domain via cname located on ali dcdn:
{"level":"debug","ts":1707068387.361652,"logger":"events","msg":"event","name":"tls_get_certificate","id":"7a934ed5-4e94-4062-a581-9c3363666c2b","origin":"tls","data":{"client_hello":{"CipherSuites":[4866,4865,4867,198,199,49196,49200,159,52393,52392,52394,49195,49199,158,49188,49192,107,49187,49191,103,49162,49172,57,49161,49171,51,157,156,61,60,53,47,255],"ServerName":"www.gkm.pw","SupportedCurves":[29,23,30,25,24,41],"SupportedPoints":"AAEC","SignatureSchemes":[1027,1283,1539,1800,2055,2056,2057,2058,2059,2052,2053,2054,1025,1281,1537,771,515,769,513,770,514,1026,1282,1538],"SupportedProtos":["http/1.1"],"SupportedVersions":[772,771,770,769],"RemoteAddr":{"IP":"163.181.15.152","Port":32968,"Zone":""},"LocalAddr":{"IP":"31.129.111.230","Port":443,"Zone":""}}}}
{"level":"debug","ts":1707068387.36311,"logger":"tls.handshake","msg":"choosing certificate","identifier":"www.gkm.pw","num_choices":1}
{"level":"debug","ts":1707068387.3633964,"logger":"tls.handshake","msg":"custom certificate selection results","identifier":"www.gkm.pw","subjects":["www.gkm.pw"],"managed":false,"issuer_key":"","hash":"7d4ead4e400ee409137f316ed7edafe561ea9517b9541ea6448e39e7fbc001a8"}
{"level":"debug","ts":1707068387.3635418,"logger":"tls.handshake","msg":"matched certificate in cache","remote_ip":"163.181.15.152","remote_port":"32968","subjects":["www.gkm.pw"],"managed":false,"expiration":1713553876,"hash":"7d4ead4e400ee409137f316ed7edafe561ea9517b9541ea6448e39e7fbc001a8"}
{"level":"debug","ts":1707068387.4141686,"logger":"http.handlers.reverse_proxy","msg":"selected upstream","dial":"127.0.0.1:15450","total_upstreams":1}
{"level":"debug","ts":1707068387.4197721,"logger":"http.handlers.reverse_proxy","msg":"upstream roundtrip","upstream":"127.0.0.1:15450","duration":0.00516051,"request":{"remote_ip":"163.181.15.152","remote_port":"32968","client_ip":"193.200.74.177","proto":"HTTP/1.1","method":"PUT","host":"gkm.pw","uri":"/akkl190219","headers":{"Accept-Encoding":["identity"],"Ali-Swift-Origin-Host":["gkm.pw"],"Ali-Swift-Log-Host":["www.gkm.pw"],"Via":["ru3.l1, l2de2.l2"],"X-Forwarded-Host":["gkm.pw"],"Ali-Tproxy-Urequest-Timeout":["30"],"Ali-Swift-Stat-Host":["www.gkm.pw"],"Ali-Cdn-Appview-Name":["cdn-tengine"],"Ali-Proxy-Is-Free":["0"],"Ali-Cdn-Real-Port":["12894"],"Ali-Cdn-Adaptive-Ports":["443,443"],"Ali-Swift-Ukeepalive-Timeout":["30"],"Eagleeye-Traceid":["2ff602a417070683872734651e"],"Ali-Swift-Urequest-Timeout":["30"],"X-Forwarded-For":["193.200.74.177, 163.181.15.152"],"Ali-Swift-Force-Ttl-Code":["400=0"],"Ali-Swift-Origin-Port":["443"],"X-Client-Scheme":["https"],"Ali-Proxy-Is-Hot":["0"],"Ali-Cdn-Real-Ip":["193.200.74.177"],"Ali-Swift-Range-Cache":["on"],"X-Alicdn-Da-Via":["47.246.2.228,163.181.15.248"],"X-Forwarded-Proto":["https"],"Ali-Swift-Origin-Scheme":["https"],"User-Agent":["Go-http-client/2.0"]},"tls":{"resumed":false,"version":772,"cipher_suite":4865,"proto":"http/1.1","server_name":"www.gkm.pw"}},"headers":{"Cache-Control":["no-store"],"Date":["Sun, 04 Feb 2024 17:39:47 GMT"]},"status":200}
{"level":"debug","ts":1707068389.9177454,"logger":"events","msg":"event","name":"tls_get_certificate","id":"e1d196a3-e110-4cd6-a959-70f59df7e8d3","origin":"tls","data":{"client_hello":{"CipherSuites":[4866,4865,4867,198,199,49196,49200,159,52393,52392,52394,49195,49199,158,49188,49192,107,49187,49191,103,49162,49172,57,49161,49171,51,157,156,61,60,53,47,255],"ServerName":"www.gkm.pw","SupportedCurves":[29,23,30,25,24,41],"SupportedPoints":"AAEC","SignatureSchemes":[1027,1283,1539,1800,2055,2056,2057,2058,2059,2052,2053,2054,1025,1281,1537,771,515,769,513,770,514,1026,1282,1538],"SupportedProtos":["http/1.1"],"SupportedVersions":[772,771,770,769],"RemoteAddr":{"IP":"163.181.15.149","Port":22230,"Zone":""},"LocalAddr":{"IP":"31.129.111.230","Port":443,"Zone":""}}}}
{"level":"debug","ts":1707068389.918107,"logger":"tls.handshake","msg":"choosing certificate","identifier":"www.gkm.pw","num_choices":1}
{"level":"debug","ts":1707068389.9182062,"logger":"tls.handshake","msg":"custom certificate selection results","identifier":"www.gkm.pw","subjects":["www.gkm.pw"],"managed":false,"issuer_key":"","hash":"7d4ead4e400ee409137f316ed7edafe561ea9517b9541ea6448e39e7fbc001a8"}
{"level":"debug","ts":1707068389.9182796,"logger":"tls.handshake","msg":"matched certificate in cache","remote_ip":"163.181.15.149","remote_port":"22230","subjects":["www.gkm.pw"],"managed":false,"expiration":1713553876,"hash":"7d4ead4e400ee409137f316ed7edafe561ea9517b9541ea6448e39e7fbc001a8"}
{"level":"debug","ts":1707068389.9658315,"logger":"http.handlers.reverse_proxy","msg":"selected upstream","dial":"127.0.0.1:15450","total_upstreams":1}
{"level":"debug","ts":1707068389.968236,"logger":"http.handlers.reverse_proxy","msg":"upstream roundtrip","upstream":"127.0.0.1:15450","duration":0.001739005,"request":{"remote_ip":"163.181.15.149","remote_port":"22230","client_ip":"193.200.74.177","proto":"HTTP/1.1","method":"PUT","host":"gkm.pw","uri":"/akkl190219","headers":{"Ali-Swift-Urequest-Timeout":["30"],"Ali-Cdn-Adaptive-Ports":["443,443"],"X-Forwarded-Proto":["https"],"X-Client-Scheme":["https"],"Via":["ru3.l1, l2de2.l2"],"Ali-Proxy-Is-Free":["0"],"Accept-Encoding":["identity"],"Ali-Swift-Log-Host":["www.gkm.pw"],"Ali-Swift-Stat-Host":["www.gkm.pw"],"Ali-Swift-Ukeepalive-Timeout":["30"],"Ali-Tproxy-Urequest-Timeout":["30"],"Ali-Swift-Force-Ttl-Code":["400=0"],"User-Agent":["Go-http-client/2.0"],"Ali-Swift-Origin-Port":["443"],"X-Alicdn-Da-Via":["47.246.2.228,163.181.15.248"],"Ali-Cdn-Real-Ip":["193.200.74.177"],"Ali-Cdn-Appview-Name":["cdn-tengine"],"Ali-Proxy-Is-Hot":["0"],"Ali-Swift-Origin-Host":["gkm.pw"],"X-Forwarded-For":["193.200.74.177, 163.181.15.149"],"Ali-Cdn-Real-Port":["4596"],"Ali-Swift-Range-Cache":["on"],"Eagleeye-Traceid":["2ff6029b17070683898417788e"],"Ali-Swift-Origin-Scheme":["https"],"X-Forwarded-Host":["gkm.pw"]},"tls":{"resumed":false,"version":772,"cipher_suite":4865,"proto":"http/1.1","server_name":"www.gkm.pw"}},"headers":{"Cache-Control":["no-store"],"Date":["Sun, 04 Feb 2024 17:39:49 GMT"]},"status":200}
{"level":"debug","ts":1707068391.2426512,"logger":"http.stdlib","msg":"http: TLS handshake error from 163.181.38.182:33896: EOF"}
{"level":"debug","ts":1707068392.0614164,"logger":"http.stdlib","msg":"http: TLS handshake error from 163.181.83.186:44374: EOF"}
{"level":"error","ts":1707068398.9574835,"logger":"http.handlers.reverse_proxy","msg":"reading from backend","error":"unexpected EOF"}
{"level":"error","ts":1707068398.9575624,"logger":"http.handlers.reverse_proxy","msg":"aborting with incomplete response","upstream":"127.0.0.1:15450","duration":0.001739005,"request":{"remote_ip":"163.181.15.149","remote_port":"22230","client_ip":"193.200.74.177","proto":"HTTP/1.1","method":"PUT","host":"gkm.pw","uri":"/akkl190219","headers":{"Ali-Swift-Urequest-Timeout":["30"],"Ali-Cdn-Adaptive-Ports":["443,443"],"X-Forwarded-Proto":["https"],"X-Client-Scheme":["https"],"Via":["ru3.l1, l2de2.l2"],"Ali-Proxy-Is-Free":["0"],"Accept-Encoding":["identity"],"Ali-Swift-Log-Host":["www.gkm.pw"],"Ali-Swift-Stat-Host":["www.gkm.pw"],"Ali-Swift-Ukeepalive-Timeout":["30"],"Ali-Tproxy-Urequest-Timeout":["30"],"Ali-Swift-Force-Ttl-Code":["400=0"],"User-Agent":["Go-http-client/2.0"],"Ali-Swift-Origin-Port":["443"],"X-Alicdn-Da-Via":["47.246.2.228,163.181.15.248"],"Ali-Cdn-Real-Ip":["193.200.74.177"],"Ali-Cdn-Appview-Name":["cdn-tengine"],"Ali-Proxy-Is-Hot":["0"],"Ali-Swift-Origin-Host":["gkm.pw"],"X-Forwarded-For":["193.200.74.177, 163.181.15.149"],"Ali-Cdn-Real-Port":["4596"],"Ali-Swift-Range-Cache":["on"],"Eagleeye-Traceid":["2ff6029b17070683898417788e"],"Ali-Swift-Origin-Scheme":["https"],"X-Forwarded-Host":["gkm.pw"]},"tls":{"resumed":false,"version":772,"cipher_suite":4865,"proto":"http/1.1","server_name":"www.gkm.pw"}},"error":"reading: unexpected EOF"}
.
3. Caddy version:
v2.7.6 h1:w0NymbG2m9PcvKWsrXO6EEkY9Ru4FJK8uQbYcev1p3A=
4. How I installed and ran Caddy:
apt install caddy
a. System environment:
Ubuntu 22.04.3 LTS (GNU/Linux 5.15.0-92-generic x86_64
b. Command:
systemctl start caddy
c. Service/unit/compose file:
[Unit]
Description=Caddy
Documentation=https://caddyserver.com/docs/
After=network.target network-online.target
Requires=network-online.target
[Service]
Type=notify
User=caddy
Group=caddy
ExecStart=/usr/bin/caddy run --environ --config /etc/caddy/Caddyfile
ExecReload=/usr/bin/caddy reload --config /etc/caddy/Caddyfile --force
TimeoutStopSec=5s
LimitNOFILE=1048576
LimitNPROC=512
PrivateTmp=true
ProtectSystem=full
AmbientCapabilities=CAP_NET_ADMIN CAP_NET_BIND_SERVICE
[Install]
WantedBy=multi-user.target
.
d. My complete Caddy config:
{
log {
level debug
output file /var/log/caddy/debug.log
}
auto_https off
servers {
trusted_proxies static 163.181.0.0/16 47.246.0.0/16 8.0.0.0/9
}
}
gkm.pw {
tls /ssl/gkm.pw.pem /ssl/gkm.pw.key.pem
tls /ssl/www.gkm.pw.pem /ssl/www.gkm.pw.key.pem
root * /var/www/html/
file_server
@http2only {
path /akkl190219
}
reverse_proxy @http2only 127.0.0.1:15450 {
transport http {
tls_server_name gkm.pw
}
}
}
