Help with mapping backend service showing 502 error

1. The problem I’m having:

When I try to access the website harry.knorrfamily.org I get an error as shown in the following logs. I am using mappings to set the hostname to a backend ip. The mappings are managed via the api using @id for each entry. I get a 502 error “Bad Gateway”

2. Error messages and/or full log output:

{
  "level": "debug",
  "ts": 1749501318.3767292,
  "logger": "http.handlers.reverse_proxy",
  "msg": "upstream roundtrip",
  "upstream": "{http.request.host}",
  "duration": 0.006308384,
  "request": {
    "remote_ip": "70.66.209.81",
    "remote_port": "55770",
    "client_ip": "70.66.209.81",
    "proto": "HTTP/2.0",
    "method": "GET",
    "host": "harry.knorrfamily.org",
    "uri": "/",
    "headers": {
      "Via": [
        "2.0 Caddy"
      ],
      "Sec-Fetch-User": [
        "?1"
      ],
      "Dnt": [
        "1"
      ],
      "Sec-Fetch-Site": [
        "none"
      ],
      "X-Forwarded-For": [
        "70.66.209.81"
      ],
      "X-Forwarded-Host": [
        "harry.knorrfamily.org"
      ],
      "Accept": [
        "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7"
      ],
      "Upgrade-Insecure-Requests": [
        "1"
      ],
      "User-Agent": [
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36"
      ],
      "Sec-Ch-Ua-Mobile": [
        "?0"
      ],
      "Sec-Fetch-Mode": [
        "navigate"
      ],
      "Accept-Language": [
        "en-US,en;q=0.9"
      ],
      "Sec-Fetch-Dest": [
        "document"
      ],
      "Accept-Encoding": [
        "gzip, deflate, br, zstd"
      ],
      "X-Forwarded-Proto": [
        "https"
      ],
      "Sec-Ch-Ua": [
        "\"Google Chrome\";v=\"137\", \"Chromium\";v=\"137\", \"Not/A)Brand\";v=\"24\""
      ],
      "Priority": [
        "u=0, i"
      ],
      "Sec-Ch-Ua-Platform": [
        "\"Windows\""
      ]
    },
    "tls": {
      "resumed": false,
      "version": 772,
      "cipher_suite": 4865,
      "proto": "h2",
      "server_name": "harry.knorrfamily.org"
    }
  },
  "error": "dial tcp 38.186.49.185:0: connect: connection refused"
}.

-----------
$ curl -vv https://harry.knorrfamily.org
14:02:36.742000 [0-0] * Host harry.knorrfamily.org:443 was resolved.
14:02:36.744000 [0-0] * IPv6: (none)
14:02:36.745000 [0-0] * IPv4: 38.186.49.185
14:02:36.746000 [0-0] * [HTTPS-CONNECT] created with 1 ALPNs -> 0
14:02:36.748000 [0-0] * [HTTPS-CONNECT] added
14:02:36.749000 [0-0] * [HTTPS-CONNECT] connect, init
14:02:36.750000 [0-0] *   Trying 38.186.49.185:443...
14:02:36.752000 [0-0] * [HTTPS-CONNECT] connect -> 0, done=0
14:02:36.753000 [0-0] * [HTTPS-CONNECT] adjust_pollset -> 1 socks
14:02:36.754000 [0-0] * [HTTPS-CONNECT] connect -> 0, done=0
14:02:36.755000 [0-0] * [HTTPS-CONNECT] adjust_pollset -> 1 socks
14:02:36.770000 [0-0] * schannel: disabled automatic use of client certificate
14:02:36.774000 [0-0] * [HTTPS-CONNECT] connect -> 0, done=0
14:02:36.775000 [0-0] * [HTTPS-CONNECT] adjust_pollset -> 1 socks
14:02:36.797000 [0-0] * [HTTPS-CONNECT] connect+handshake h1: 46ms, 1st data: 44ms
14:02:36.798000 [0-0] * [HTTPS-CONNECT] connect -> 0, done=1
14:02:36.800000 [0-0] * Connected to harry.knorrfamily.org (38.186.49.185) port 443
14:02:36.801000 [0-0] * using HTTP/1.x
14:02:36.802000 [0-0] > GET / HTTP/1.1
14:02:36.802000 [0-0] > Host: harry.knorrfamily.org
14:02:36.802000 [0-0] > User-Agent: curl/8.12.1
14:02:36.802000 [0-0] > Accept: */*
14:02:36.802000 [0-0] >
14:02:36.808000 [0-0] * schannel: remote party requests renegotiation
14:02:36.810000 [0-0] * schannel: renegotiating SSL/TLS connection
14:02:36.811000 [0-0] * schannel: SSL/TLS connection renegotiated
14:02:36.812000 [0-0] * Request completely sent off
14:02:36.833000 [0-0] < HTTP/1.1 502 Bad Gateway
14:02:36.835000 [0-0] < Alt-Svc: h3=":443"; ma=2592000
14:02:36.836000 [0-0] < Server: Caddy
14:02:36.837000 [0-0] < Date: Mon, 09 Jun 2025 21:02:37 GMT
14:02:36.838000 [0-0] < Content-Length: 0
14:02:36.839000 [0-0] <
14:02:36.840000 [0-0] * Connection #0 to host harry.knorrfamily.org left intact

3. Caddy version:

admin@caddy1:~$ caddy version
v2.10.0 h1:fonubSaQKF1YANl8TXqGcn4IbIRUDdfAkpcsfI/vX5U=

4. How I installed and ran Caddy:

apt install -y debian-keyring debian-archive-keyring apt-transport-https curl -y

curl -1sLf 'https://dl.cloudsmith.io/public/caddy/stable/gpg.key' | sudo gpg --dearmor -o /usr/share/keyrings/caddy-stable-archive-keyring.gpg

curl -1sLf 'https://dl.cloudsmith.io/public/caddy/stable/debian.deb.txt' | sudo tee /etc/apt/sources.list.d/caddy-stable.list

apt update

apt install caddy -y
caddy add-package github.com/caddy-dns/cloudflare


a. System environment:

Linux caddy1 6.1.0-32-cloud-amd64 #1 SMP PREEMPT_DYNAMIC Debian 6.1.129-1 (2025-03-06) x86_64 GNU/Linux

b. Command:

c. Service/unit/compose file:

root@caddy0:/home/admin# cat /lib/systemd/system/caddy-api.service
# caddy-api.service
#
# For using Caddy with its API.
#
# This unit is "durable" in that it will automatically resume
# the last active configuration if the service is restarted.
#
# See https://caddyserver.com/docs/install for instructions.

[Unit]
Description=Caddy
Documentation=https://caddyserver.com/docs/
After=network.target network-online.target
Requires=network-online.target

[Service]
Type=notify
User=caddy
Group=caddy
ExecStart=/usr/bin/caddy run --environ --resume
TimeoutStopSec=5s
LimitNOFILE=1048576
PrivateTmp=true
ProtectSystem=full
AmbientCapabilities=CAP_NET_ADMIN CAP_NET_BIND_SERVICE

[Install]
WantedBy=multi-user.target

d. My complete Caddy config:

{
    "admin": {
        "listen": "192.168.0.2:4334"
    },
    "apps": {
        "http": {
            "http_port": 80,
            "https_port": 443,
            "metrics": {},
            "servers": {
                "srv0": {
                    "errors": {
                        "routes": [
                            {
                                "handle": [
                                    {
                                        "handler": "subroute",
                                        "routes": [
                                            {
                                                "handle": [
                                                    {
                                                        "handler": "file_server",
                                                        "hide": [
                                                            "/etc/caddy/Caddyfile"
                                                        ],
                                                        "index_names": [
                                                            "/5xx_error.html"
                                                        ],
                                                        "root": "/usr/share/caddy/error_pages/game_servers"
                                                    }
                                                ],
                                                "match": [
                                                    {
                                                        "expression": "{http.error.status_code} >= 500 && {http.error.status_code} <= 599"
                                                    }
                                                ]
                                            }
                                        ]
                                    }
                                ],
                                "match": [
                                    {
                                        "host": [
                                            "*.domain1.com",
                                            "*.domain2.com",
                                            "*.domain3.com",
                                            "*.knorrfamily.org"
                                        ]
                                    }
                                ],
                                "terminal": true
                            }
                        ]
                    },
                    "listen": [
                        "38.186.49.185:443"
                    ],
                    "routes": [
                        {
                            "handle": [
                                {
                                    "handler": "subroute",
                                    "routes": [
                                        {
                                            "@id": "domain1.com",
                                            "handle": [
                                                {
                                                    "destinations": [
                                                        "{backend_ip}"
                                                    ],
                                                    "handler": "map",
                                                    "mappings": [],
                                                    "source": "{http.request.host}"
                                                },
                                                {
                                                    "handler": "subroute",
                                                    "routes": [
                                                        {
                                                            "handle": [
                                                                {
                                                                    "handler": "reverse_proxy",
                                                                    "upstreams": [
                                                                        {
                                                                            "dial": "{http.request.host}"
                                                                        },
                                                                        {
                                                                            "dial": "{backend_ip}:3030"
                                                                        }
                                                                    ]
                                                                }
                                                            ],
                                                            "match": [
                                                                {
                                                                    "path": [
                                                                        "/webdav/*"
                                                                    ]
                                                                }
                                                            ]
                                                        },
                                                        {
                                                            "handle": [
                                                                {
                                                                    "handler": "reverse_proxy",
                                                                    "stream_close_delay": 28800000000000,
                                                                    "upstreams": [
                                                                        {
                                                                            "dial": "{http.request.host}"
                                                                        },
                                                                        {
                                                                            "dial": "{backend_ip}:30000"
                                                                        }
                                                                    ]
                                                                }
                                                            ]
                                                        }
                                                    ]
                                                }
                                            ]
                                        },
                                        {
                                            "@id": "domain3.com",
                                            "handle": [
                                                {
                                                    "destinations": [
                                                        "{backend_ip}"
                                                    ],
                                                    "handler": "map",
                                                    "mappings": [],
                                                    "source": "{http.request.host}"
                                                },
                                                {
                                                    "handler": "subroute",
                                                    "routes": [
                                                        {
                                                            "handle": [
                                                                {
                                                                    "handler": "reverse_proxy",
                                                                    "upstreams": [
                                                                        {
                                                                            "dial": "{http.request.host}"
                                                                        },
                                                                        {
                                                                            "dial": "{backend_ip}:3030"
                                                                        }
                                                                    ]
                                                                }
                                                            ],
                                                            "match": [
                                                                {
                                                                    "path": [
                                                                        "/webdav/*"
                                                                    ]
                                                                }
                                                            ]
                                                        },
                                                        {
                                                            "handle": [
                                                                {
                                                                    "handler": "reverse_proxy",
                                                                    "stream_close_delay": 28800000000000,
                                                                    "upstreams": [
                                                                        {
                                                                            "dial": "{http.request.host}"
                                                                        },
                                                                        {
                                                                            "dial": "{backend_ip}:30000"
                                                                        }
                                                                    ]
                                                                }
                                                            ]
                                                        }
                                                    ]
                                                }
                                            ]
                                        },
                                        {
                                            "@id": "domain2.com",
                                            "handle": [
                                                {
                                                    "destinations": [
                                                        "{backend_ip}"
                                                    ],
                                                    "handler": "map",
                                                    "mappings": [],
                                                    "source": "{http.request.host}"
                                                },
                                                {
                                                    "handler": "subroute",
                                                    "routes": [
                                                        {
                                                            "handle": [
                                                                {
                                                                    "handler": "reverse_proxy",
                                                                    "upstreams": [
                                                                        {
                                                                            "dial": "{http.request.host}"
                                                                        },
                                                                        {
                                                                            "dial": "{backend_ip}:3030"
                                                                        }
                                                                    ]
                                                                }
                                                            ],
                                                            "match": [
                                                                {
                                                                    "path": [
                                                                        "/webdav/*"
                                                                    ]
                                                                }
                                                            ]
                                                        },
                                                        {
                                                            "handle": [
                                                                {
                                                                    "handler": "reverse_proxy",
                                                                    "stream_close_delay": 28800000000000,
                                                                    "upstreams": [
                                                                        {
                                                                            "dial": "{http.request.host}"
                                                                        },
                                                                        {
                                                                            "dial": "{backend_ip}:30000"
                                                                        }
                                                                    ]
                                                                }
                                                            ]
                                                        }
                                                    ]
                                                }
                                            ]
                                        },
                                        {
                                            "@id": "knorrfamily.org",
                                            "handle": [
                                                {
                                                    "destinations": [
                                                        "{backend_ip}"
                                                    ],
                                                    "handler": "map",
                                                    "mappings": [
                                                        {
                                                            "@id": "harry",
                                                            "input": "harry.knorrfamily.org",
                                                            "outputs": [
                                                                "192.168.255.237"
                                                            ]
                                                        }
                                                    ],
                                                    "source": "{http.request.host}"
                                                },
                                                {
                                                    "handler": "subroute",
                                                    "routes": [
                                                        {
                                                            "handle": [
                                                                {
                                                                    "handler": "reverse_proxy",
                                                                    "upstreams": [
                                                                        {
                                                                            "dial": "{http.request.host}"
                                                                        },
                                                                        {
                                                                            "dial": "{backend_ip}:3030"
                                                                        }
                                                                    ]
                                                                }
                                                            ],
                                                            "match": [
                                                                {
                                                                    "path": [
                                                                        "/webdav/*"
                                                                    ]
                                                                }
                                                            ]
                                                        },
                                                        {
                                                            "handle": [
                                                                {
                                                                    "handler": "reverse_proxy",
                                                                    "stream_close_delay": 28800000000000,
                                                                    "upstreams": [
                                                                        {
                                                                            "dial": "{http.request.host}"
                                                                        },
                                                                        {
                                                                            "dial": "{backend_ip}:30000"
                                                                        }
                                                                    ]
                                                                }
                                                            ]
                                                        }
                                                    ]
                                                }
                                            ]
                                        }
                                    ]
                                }
                            ],
                            "match": [
                                {
                                    "host": [
                                        "*.domain1.com",
                                        "*.domain3.com",
                                        "*.domain2.com",
                                        "*.knorrfamily.org"
                                    ]
                                }
                            ],
                            "terminal": true
                        }
                    ]
                }
            }
        },
        "tls": {
            "automation": {
                "policies": [
                    {
                        "issuers": [
                            {
                                "challenges": {
                                    "dns": {
                                        "provider": {
                                            "api_token": "<redacted>",
                                            "name": "cloudflare"
                                        }
                                    },
                                    "http": {
                                        "alternate_port": 80
                                    },
                                    "tls-alpn": {
                                        "alternate_port": 443
                                    }
                                },
                                "email": "admin@domain3.com",
                                "module": "acme"
                            }
                        ],
                        "subjects": [
                            "*.domain3.com",
                            "*.domain2.com",
                            "*.domain1.com",
                            "*.knorrfamily.org"
                        ]
                    }
                ]
            }
        }
    },
    "logging": {
        "logs": {
            "default": {
                "level": "DEBUG"
            }
        }
    }
}

5. Links to relevant resources:

To access the url in question https://harry.knorrfamily.org. I have confirmed that the backend is responding via curl, from the host that is running caddy.

You should see selected upstream and upstream roundtrip log messages that look like this:

2025/06/09 21:26:17.730 DEBUG   http.handlers.reverse_proxy     selected upstream       {"dial": "localhost:8080", "total_upstreams": 1}
2025/06/09 21:26:17.731 DEBUG   http.handlers.reverse_proxy     upstream roundtrip      {"upstream": "localhost:8080", "duration": 0.0012489, "request": {"remote_ip": "::1", "remote_port": "57484", "client_ip": "::1", "proto": "HTTP/2.0", "method": "GET", "host": "localhost", "uri": "/", "headers": {"X-Forwarded-Proto": ["https"], "X-Forwarded-Host": ["localhost"], "Via": ["2.0 Caddy"], "User-Agent": ["curl/8.14.1"], "Accept": ["*/*"], "X-Forwarded-For": ["::1"]}, "tls": {"resumed": false, "version": 772, "cipher_suite": 4865, "proto": "h2", "server_name": "localhost"}}, "error": "dial tcp [::1]:8080: connect: connection refused"}
2025/06/09 21:26:17.731 ERROR   http.log.error  dial tcp [::1]:8080: connect: connection refused        {"request": {"remote_ip": "::1", "remote_port": "57484", "client_ip": "::1", "proto": "HTTP/2.0", "method": "GET", "host": "localhost", "uri": "/", "headers": {"User-Agent": ["curl/8.14.1"], "Accept": ["*/*"]}, "tls": {"resumed": false, "version": 772, "cipher_suite": 4865, "proto": "h2", "server_name": "localhost"}}, "duration": 0.001752473, "status": 502, "err_id": "bvn578s28", "err_trace": "reverseproxy.statusError (reverseproxy.go:1390)"}

Here is the missing output.

{"level":"debug","ts":1749506659.1470582,"logger":"http.handlers.reverse_proxy","msg":"selected upstream","dial":"harry.knorrfamily.org:0","total_upstreams":2}```

 {
  "level": "debug",
  "ts": 1749506659.1546988,
  "logger": "http.handlers.reverse_proxy",
  "msg": "upstream roundtrip",
  "upstream": "{http.request.host}",
  "duration": 0.005974485,
  "request": {
    "remote_ip": "70.66.209.81",
    "remote_port": "63594",
    "client_ip": "70.66.209.81",
    "proto": "HTTP/2.0",
    "method": "GET",
    "host": "harry.knorrfamily.org",
    "uri": "/",
    "headers": {
      "Sec-Ch-Ua-Platform": [
        "\"Windows\""
      ],
      "X-Forwarded-For": [
        "70.66.209.81"
      ],
      "X-Forwarded-Proto": [
        "https"
      ],
      "Sec-Fetch-Mode": [
        "navigate"
      ],
      "Sec-Fetch-User": [
        "?1"
      ],
      "Sec-Fetch-Dest": [
        "document"
      ],
      "Sec-Ch-Ua": [
        "\"Google Chrome\";v=\"137\", \"Chromium\";v=\"137\", \"Not/A)Brand\";v=\"24\""
      ],
      "Sec-Ch-Ua-Mobile": [
        "?0"
      ],
      "X-Forwarded-Host": [
        "harry.knorrfamily.org"
      ],
      "Via": [
        "2.0 Caddy"
      ],
      "Upgrade-Insecure-Requests": [
        "1"
      ],
      "User-Agent": [
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36"
      ],
      "Accept-Encoding": [
        "gzip, deflate, br, zstd"
      ],
      "Accept": [
        "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7"
      ],
      "Sec-Fetch-Site": [
        "none"
      ],
      "Priority": [
        "u=0, i"
      ],
      "Accept-Language": [
        "en-US,en;q=0.9"
      ],
      "Dnt": [
        "1"
      ]
    },
    "tls": {
      "resumed": true,
      "version": 772,
      "cipher_suite": 4865,
      "proto": "h2",
      "server_name": "harry.knorrfamily.org"
    }
  },
  "error": "dial tcp 38.186.49.185:0: connect: connection refused"
}

There it is:

The port 0 is the issue.