1. The problem I’m having:
I am using postman to make https (get & post) requests to my application. Caddy is the proxy service sitting in front of my applications.
When I make these requests I get status 200 about 20% of the time with status 502 the remaining 80% of the time.
The url I am hitting via GET is https://helcim.foundryserver.ca/health-check
I did provide a custom message in my application to ensure I am getting the correct status 200 message at the right location in the code.
2. Error messages and/or full log output:
Successful Log Entry
Mar 14 23:01:34 caddy1 caddy[368]: {"level":"debug","ts":1741993294.3780296,"logger":"http.handlers.reverse_proxy","msg":"selected upstream","dial":"nodeport.k0s.local:31020","total_upstreams":2}
Mar 14 23:01:34 caddy1 caddy[368]: {"level":"debug","ts":1741993294.3906019,"logger":"http.handlers.reverse_proxy","msg":"upstream roundtrip","upstream":"{backend_ip}","duration":0.012401336,"request":{"remote_ip":"70.66.209.81","remote_port":"51443","client_ip":"70.66.209.81","proto":"HTTP/1.1","method":"GET","host":"helcim.foundryserver.ca","uri":"/health-check","headers":{"User-Agent":["PostmanRuntime/7.43.0"],"Accept":["*/*"],"Cache-Control":["no-cache"],"Postman-Token":["155bd9ec-5d2d-4d3c-9b7f-f94b34e7e73b"],"X-Forwarded-Proto":["https"],"X-Forwarded-Host":["helcim.foundryserver.ca"],"Accept-Encoding":["gzip, deflate, br"],"X-Forwarded-For":["70.66.209.81"]},"tls":{"resumed":false,"version":772,"cipher_suite":4865,"proto":"","server_name":"helcim.foundryserver.ca"}},"headers":{"Content-Length":["2"],"Etag":["W/\"2-nOO9QiTIwXgNtWtBJezz8kv3SLc\""],"Date":["Fri, 14 Mar 2025 23:01:34 GMT"],"Connection":["keep-alive"],"Keep-Alive":["timeout=5"],"X-Powered-By":["Express"],"Content-Type":["text/plain; charset=utf-8"]},"status":200}
Not Successful Entry
Mar 14 23:01:28 caddy1 caddy[368]: {"level":"debug","ts":1741993288.9747758,"logger":"http.handlers.reverse_proxy","msg":"selected upstream","dial":"helcim.foundryserver.ca:0","total_upstreams":2}
Mar 14 23:01:31 caddy1 caddy[368]: {"level":"debug","ts":1741993291.97533,"logger":"http.handlers.reverse_proxy","msg":"upstream roundtrip","upstream":"{http.request.host}","duration":3.000437539,"request":{"remote_ip":"70.66.209.81","remote_port":"51443","client_ip":"70.66.209.81","proto":"HTTP/1.1","method":"GET","host":"helcim.foundryserver.ca","uri":"/health-check","headers":{"X-Forwarded-Proto":["https"],"X-Forwarded-Host":["helcim.foundryserver.ca"],"User-Agent":["PostmanRuntime/7.43.0"],"Accept":["*/*"],"Cache-Control":["no-cache"],"Postman-Token":["619397c8-8e17-4d5e-bc4d-86f335b98f4d"],"Accept-Encoding":["gzip, deflate, br"],"X-Forwarded-For":["70.66.209.81"]},"tls":{"resumed":false,"version":772,"cipher_suite":4865,"proto":"","server_name":"helcim.foundryserver.ca"}},"error":"dial tcp 38.186.49.166:0: i/o timeout"}
Mar 14 23:01:31 caddy1 caddy[368]: {"level":"debug","ts":1741993291.975675,"logger":"http.handlers.file_server","msg":"sanitized path join","site_root":"/usr/share/caddy/error_pages/foundryserver","fs":"","request_path":"/health-check","result":"/usr/share/caddy/error_pages/foundryserver/health-check"}
Mar 14 23:01:31 caddy1 caddy[368]: {"level":"error","ts":1741993291.9757574,"logger":"http.log.error","msg":"error handling handler error","request":{"remote_ip":"70.66.209.81","remote_port":"51443","client_ip":"70.66.209.81","proto":"HTTP/1.1","method":"GET","host":"helcim.foundryserver.ca","uri":"/health-check","headers":{"User-Agent":["PostmanRuntime/7.43.0"],"Accept":["*/*"],"Cache-Control":["no-cache"],"Postman-Token":["619397c8-8e17-4d5e-bc4d-86f335b98f4d"],"Accept-Encoding":["gzip, deflate, br"],"Connection":["keep-alive"]},"tls":{"resumed":false,"version":772,"cipher_suite":4865,"proto":"","server_name":"helcim.foundryserver.ca"}},"duration":3.000790373,"error":"{id=549n86ate} fileserver.(*FileServer).notFound (staticfiles.go:705): HTTP 404","first_error":{"msg":"dial tcp 38.186.49.166:0: i/o timeout","status":502,"err_id":"4u4bct8wb","err_trace":"reverseproxy.statusError (reverseproxy.go:1373)"}}
3. Caddy version:
v2.9.1 h1:OEYiZ7DbCzAWVb6TNEkjRcSCRGHVoZsJinoDR/n9oaY=
4. How I installed and ran Caddy:
apt install -y debian-keyring debian-archive-keyring apt-transport-https curl -y
curl -1sLf 'https://dl.cloudsmith.io/public/caddy/stable/gpg.key' | sudo gpg --dearmor -o /usr/share/keyrings/caddy-stable-archive-keyring.gpg
curl -1sLf 'https://dl.cloudsmith.io/public/caddy/stable/debian.deb.txt' | sudo tee /etc/apt/sources.list.d/caddy-stable.list
apt update
apt install caddy -y
caddy add-package github.com/caddy-dns/cloudflare
Change caddy service file.
systemctl stop caddy
systemctl disable caddy
systemctl enable caddy-api
systemctl start caddy-api
a. System environment:
Debian 12 on a proxmox VM
c. Service/unit/compose file:
# caddy-api.service
#
# For using Caddy with its API.
#
# This unit is "durable" in that it will automatically resume
# the last active configuration if the service is restarted.
#
# See https://caddyserver.com/docs/install for instructions.
[Unit]
Description=Caddy
Documentation=https://caddyserver.com/docs/
After=network.target network-online.target
Requires=network-online.target
[Service]
Type=notify
User=caddy
Group=caddy
ExecStart=/usr/bin/caddy run --environ --resume
TimeoutStopSec=5s
LimitNOFILE=1048576
PrivateTmp=true
ProtectSystem=full
AmbientCapabilities=CAP_NET_ADMIN CAP_NET_BIND_SERVICE
[Install]
WantedBy=multi-user.target
d. My complete Caddy config:
{
"admin": { "listen": "0.0.0.0:4334" },
"apps": {
"http": {
"http_port": 8080,
"https_port": 8443,
"metrics": {},
"servers": {
"srv0": {
"errors": {
"routes": [
{
"handle": [
{
"handler": "subroute",
"routes": [
{
"handle": [
{
"handler": "file_server",
"hide": ["/etc/caddy/Caddyfile"],
"index_names": ["/5xx_error.html"],
"root": "/usr/share/caddy/error_pages/foundryserver"
}
],
"match": [
{
"expression": "{http.error.status_code} \u003e= 500 \u0026\u0026 {http.error.status_code} \u003c= 599"
}
]
}
]
}
],
"match": [
{
"host": [
"foundryserver.ca",
"www.foundryserver.ca",
"helcim.foundryserver.ca",
"max.foundryserver.ca"
]
}
],
"terminal": true
},
{
"handle": [
{
"handler": "subroute",
"routes": [
{
"handle": [
{
"handler": "file_server",
"hide": ["/etc/caddy/Caddyfile"],
"index_names": ["/5xx_error.html"],
"root": "/usr/share/caddy/error_pages/game_servers"
}
],
"match": [
{
"expression": "{http.error.status_code} \u003e= 500 \u0026\u0026 {http.error.status_code} \u003c= 599"
}
]
}
]
}
],
"match": [{ "host": ["*.foundryserver.ca"] }],
"terminal": true
}
]
},
"listen": [":8443"],
"routes": [
{
"handle": [
{
"handler": "subroute",
"routes": [
{
"handle": [
{
"destinations": ["{backend_ip}"],
"handler": "map",
"mappings": [
{ "input": "foundryserver.ca", "outputs": ["nodeport.k0s.local:31000"] },
{ "input": "www.foundryserver.ca", "outputs": ["nodeport.k0s.local:31000"] },
{ "input": "helcim.foundryserver.ca", "outputs": ["nodeport.k0s.local:31020"] },
{ "input": "max.foundryserver.ca", "outputs": ["nodeport.k0s.local:31010"] }
],
"source": "{http.request.host}"
},
{
"handler": "reverse_proxy",
"upstreams": [{ "dial": "{http.request.host}" }, { "dial": "{backend_ip}" }]
}
]
}
]
}
],
"match": [
{
"host": [
"foundryserver.ca",
"www.foundryserver.ca",
"helcim.foundryserver.ca",
"max.foundryserver.ca"
]
}
],
"terminal": true
},
{
"handle": [
{
"handler": "subroute",
"routes": [
{
"@id": "foundryserver.ca",
"handle": [
{
"destinations": ["{backend_ip}"],
"handler": "map",
"mappings": [
{ "@id": "bobby", "input": "bobby.foundryserver.ca", "outputs": ["192.168.217.111"] },
{ "@id": "demo", "input": "demo.foundryserver.ca", "outputs": ["192.168.60.134"] }
],
"source": "{http.request.host}"
},
{
"handler": "subroute",
"routes": [
{
"match": [
{
"path": ["/webdav/*"]
}
],
"handle": [
{
"handler": "reverse_proxy",
"upstreams": [{ "dial": "{http.request.host}" }, { "dial": "{backend_ip}:3030" }]
}
]
},
{
"handle": [
{
"handler": "reverse_proxy",
"stream_close_delay": 28800000000000,
"upstreams": [{ "dial": "{http.request.host}" }, { "dial": "{backend_ip}:30000" }]
}
]
}
]
}
]
}
]
}
],
"match": [{ "host": ["*.foundryserver.ca"] }],
"terminal": true
}
]
}
}
},
"tls": {
"automation": {
"policies": [
{
"issuers": [
{
"challenges": {
"dns": {
"provider": { "api_token": "<redacted>", "name": "cloudflare" }
},
"http": { "alternate_port": 8080 },
"tls-alpn": { "alternate_port": 8443 }
},
"email": "admin@foundryserver.com",
"module": "acme"
}
],
"subjects": [
"foundryserver.ca",
"max.foundryserver.ca",
"helcim.foundryserver.ca",
"www.foundryserver.ca",
"*.foundryserver.ca"
]
}
]
}
}
},
"logging": { "logs": { "default": { "level": "DEBUG" } } }
}