1. The problem I’m having:
We have a Rails application with a page that takes >15 seconds to render. When our users hit this page their browser times out at 15 seconds and responds with a 504 status and a message of “upstream request timeout”.
I tried increasing the “response_header_timeout” to 60s but that didn’t work. The request still times out at 15 seconds with the same status and message.
2. Error messages and/or full log output:
upstream request timeout
3. Caddy version:
v2.7.6 h1:w0NymbG2m9PcvKWsrXO6EEkY9Ru4FJK8uQbYcev1p3A=
4. How I installed and ran Caddy:
Caddy is being used as a reverse proxy and is running in an AWS ECS container.
a. System environment:
AWS ECS container
b. Command:
caddy run --config /etc/caddy/Caddyfile
c. Service/unit/compose file:
FROM caddy:2.7
COPY ./config/Caddyfile /etc/caddy/Caddyfile
CMD ["caddy", "run", "--config", "/etc/caddy/Caddyfile"]
d. My complete Caddy config:
# Global config options
# https://caddyserver.com/docs/caddyfile/options
{
# turn off admin API endpoint
admin off
servers {
trusted_proxies static private_ranges
# enabled metics
# https://caddyserver.com/docs/metrics
metrics
}
}
# Internal endpoint for serving healthchecks etc.
:80 {
respond /up "UP" 200
}
:3000 {
# Block API calls from the WWW
@blocked {
header x-forwarded-proto https
path /internal_api
}
respond @blocked 401
reverse_proxy cobalt2.{$LH_DATASET}:3000 {
# Backend API must respond to an individual API call under 60 seconds
transport http {
response_header_timeout 60s
}
}
log {
output stdout
format json
}
}
:3001 {
# Block API calls from the WWW
@blocked {
header x-forwarded-proto https
path /internal_api
}
respond @blocked 401
reverse_proxy durin.{$LH_DATASET}:3001 {
# Backend API must respond to an individual API call under 60 seconds
transport http {
response_header_timeout 60s
}
}
log {
output stdout
format json
}
}
5. Caddy log output:
{
"level": "error",
"ts": 1715094351.0666327,
"logger": "http.log.access.log0",
"msg": "handled request",
"request": {
"remote_ip": "10.202.53.160",
"remote_port": "10956",
"client_ip": "10.202.40.229",
"proto": "HTTP/1.1",
"method": "GET",
"host": "my.application.example",
"uri": "/users",
"headers": {
"Priority": [
"u=0, i"
],
"Cookie": [],
"Sec-Ch-Ua": [
"\"Chromium\";v=\"124\", \"Google Chrome\";v=\"124\", \"Not-A.Brand\";v=\"99\""
],
"Sec-Ch-Ua-Mobile": [
"?0"
],
"Upgrade-Insecure-Requests": [
"1"
],
"Accept-Language": [
"en-US,en;q=0.9"
],
"Sec-Fetch-User": [
"?1"
],
"Accept-Encoding": [
"gzip, deflate, br, zstd"
],
"Sec-Gpc": [
"1"
],
"X-Forwarded-For": [
"10.202.40.229"
],
"X-Amzn-Trace-Id": [
"Root=1-663a4340-1f7b02544e853b6b2891eda9"
],
"User-Agent": [
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
],
"Sec-Fetch-Mode": [
"navigate"
],
"Sec-Fetch-Site": [
"same-origin"
],
"Sec-Fetch-Dest": [
"document"
],
"Referer": [
"https://my.application.example/programs"
],
"X-Forwarded-Proto": [
"https"
],
"X-Forwarded-Port": [
"443"
],
"Sec-Ch-Ua-Platform": [
"\"macOS\""
],
"Accept": [
"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7"
]
}
},
"bytes_read": 0,
"user_id": "",
"duration": 15.001381496,
"size": 24,
"status": 504,
"resp_headers": {
"Server": [
"Caddy",
"envoy"
],
"Content-Length": [
"24"
],
"Content-Type": [
"text/plain"
],
"Date": [
"Tue, 07 May 2024 15:05:51 GMT"
],
"X-Envoy-Upstream-Service-Time": [
"15000"
]
}
}