1. The problem I’m having:
I’m trying to setup a caddy reverse_proxy to a php server for an API. Intended behavior: My clientside app lives at fake-aws.leadingreach.localhost:8093. That application is served properly (see config below). However, when I make a request to fake-aws.leadingreach.localhost:8093/proxy/api, I want that request reverse proxied to :15001, which should be my PHP backend which does a php_fastcgi to port 14999.
2. Error messages and/or full log output:
This is what I see in the caddy logs when I try to hit :8093/proxy/api/login_validate:
2024/10/09 15:53:21.854 DEBUG events event {"name": "tls_get_certificate", "id": "d4c07b46-fa8b-4f86-aa11-ba4f8369e512", "origin": "tls", "data": {"client_hello":{"CipherSuites":[64250,4865,4866,4867,49195,49199,49196,49200,52393,52392,49171,49172,156,157,47,53],"ServerName":"fake-aws.leadingreach.localhost","SupportedCurves":[47802,25497,29,23,24],"SupportedPoints":"AA==","SignatureSchemes":[1027,2052,1025,1283,2053,1281,2054,1537],"SupportedProtos":["h2","http/1.1"],"SupportedVersions":[47802,772,771],"RemoteAddr":{"IP":"192.168.65.1","Port":63970,"Zone":""},"LocalAddr":{"IP":"172.17.0.3","Port":8083,"Zone":""}}}}
2024/10/09 15:53:21.858 DEBUG tls.handshake choosing certificate {"identifier": "fake-aws.leadingreach.localhost", "num_choices": 1}
2024/10/09 15:53:21.859 DEBUG tls.handshake default certificate selection results {"identifier": "fake-aws.leadingreach.localhost", "subjects": ["fake-aws.leadingreach.localhost"], "managed": true, "issuer_key": "local", "hash": "3c02d41688f8745373cf85910877eb0401766991326185808fb110630a3a1ebe"}
2024/10/09 15:53:21.860 DEBUG tls.handshake matched certificate in cache {"remote_ip": "192.168.65.1", "remote_port": "63970", "subjects": ["fake-aws.leadingreach.localhost"], "managed": true, "expiration": "2024/10/10 03:52:06.000", "hash": "3c02d41688f8745373cf85910877eb0401766991326185808fb110630a3a1ebe"}
2024/10/09 15:53:21.876 DEBUG http.stdlib http: TLS handshake error from 192.168.65.1:63970: remote error: tls: unknown certificate
2024/10/09 15:53:23.978 DEBUG events event {"name": "tls_get_certificate", "id": "94f76c1b-318e-417f-bd29-168554cc459d", "origin": "tls", "data": {"client_hello":{"CipherSuites":[27242,4865,4866,4867,49195,49199,49196,49200,52393,52392,49171,49172,156,157,47,53],"ServerName":"fake-aws.leadingreach.localhost","SupportedCurves":[19018,25497,29,23,24],"SupportedPoints":"AA==","SignatureSchemes":[1027,2052,1025,1283,2053,1281,2054,1537],"SupportedProtos":["h2","http/1.1"],"SupportedVersions":[10794,772,771],"RemoteAddr":{"IP":"192.168.65.1","Port":47602,"Zone":""},"LocalAddr":{"IP":"172.17.0.3","Port":8083,"Zone":""}}}}
2024/10/09 15:53:23.978 DEBUG tls.handshake choosing certificate {"identifier": "fake-aws.leadingreach.localhost", "num_choices": 1}
2024/10/09 15:53:23.978 DEBUG tls.handshake default certificate selection results {"identifier": "fake-aws.leadingreach.localhost", "subjects": ["fake-aws.leadingreach.localhost"], "managed": true, "issuer_key": "local", "hash": "3c02d41688f8745373cf85910877eb0401766991326185808fb110630a3a1ebe"}
2024/10/09 15:53:23.978 DEBUG tls.handshake matched certificate in cache {"remote_ip": "192.168.65.1", "remote_port": "47602", "subjects": ["fake-aws.leadingreach.localhost"], "managed": true, "expiration": "2024/10/10 03:52:06.000", "hash": "3c02d41688f8745373cf85910877eb0401766991326185808fb110630a3a1ebe"}
2024/10/09 15:53:23.979 DEBUG http.stdlib http: TLS handshake error from 192.168.65.1:47602: remote error: tls: unknown certificate
2024/10/09 15:53:23.984 DEBUG events event {"name": "tls_get_certificate", "id": "eed81916-3d62-4b6b-97b2-428d13eea5a0", "origin": "tls", "data": {"client_hello":{"CipherSuites":[64250,4865,4866,4867,49195,49199,49196,49200,52393,52392,49171,49172,156,157,47,53],"ServerName":"fake-aws.leadingreach.localhost","SupportedCurves":[64250,25497,29,23,24],"SupportedPoints":"AA==","SignatureSchemes":[1027,2052,1025,1283,2053,1281,2054,1537],"SupportedProtos":["h2","http/1.1"],"SupportedVersions":[51914,772,771],"RemoteAddr":{"IP":"192.168.65.1","Port":64025,"Zone":""},"LocalAddr":{"IP":"172.17.0.3","Port":8083,"Zone":""}}}}
2024/10/09 15:53:23.984 DEBUG tls.handshake choosing certificate {"identifier": "fake-aws.leadingreach.localhost", "num_choices": 1}
2024/10/09 15:53:23.984 DEBUG tls.handshake default certificate selection results {"identifier": "fake-aws.leadingreach.localhost", "subjects": ["fake-aws.leadingreach.localhost"], "managed": true, "issuer_key": "local", "hash": "3c02d41688f8745373cf85910877eb0401766991326185808fb110630a3a1ebe"}
2024/10/09 15:53:23.984 DEBUG tls.handshake matched certificate in cache {"remote_ip": "192.168.65.1", "remote_port": "64025", "subjects": ["fake-aws.leadingreach.localhost"], "managed": true, "expiration": "2024/10/10 03:52:06.000", "hash": "3c02d41688f8745373cf85910877eb0401766991326185808fb110630a3a1ebe"}
2024/10/09 15:53:24.031 DEBUG http.handlers.rewrite rewrote request {"request": {"remote_ip": "192.168.65.1", "remote_port": "64025", "client_ip": "192.168.65.1", "proto": "HTTP/2.0", "method": "GET", "host": "fake-aws.leadingreach.localhost:8083", "uri": "/", "headers": {"Sec-Fetch-Dest": ["document"], "Accept-Encoding": ["gzip, deflate, br, zstd"], "User-Agent": ["Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36"], "Sec-Ch-Ua": ["\"Google Chrome\";v=\"129\", \"Not=A?Brand\";v=\"8\", \"Chromium\";v=\"129\""], "Upgrade-Insecure-Requests": ["1"], "Accept": ["text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7"], "Cache-Control": ["max-age=0"], "Sec-Ch-Ua-Platform": ["\"macOS\""], "Sec-Fetch-Site": ["none"], "Sec-Fetch-Mode": ["navigate"], "Sec-Fetch-User": ["?1"], "Sec-Ch-Ua-Mobile": ["?0"], "Priority": ["u=0, i"], "Accept-Language": ["en-US,en;q=0.9"]}, "tls": {"resumed": false, "version": 772, "cipher_suite": 4865, "proto": "h2", "server_name": "fake-aws.leadingreach.localhost"}}, "method": "GET", "uri": "/index.html"}
2024/10/09 15:53:24.034 DEBUG http.handlers.file_server sanitized path join {"site_root": "/var/www/apphead/current/build_admin", "fs": "", "request_path": "/index.html", "result": "/var/www/apphead/current/build_admin/index.html"}
2024/10/09 15:53:24.035 DEBUG http.handlers.file_server opening file {"filename": "/var/www/apphead/current/build_admin/index.html"}
2024/10/09 15:53:24.076 DEBUG http.handlers.file_server sanitized path join {"site_root": "/var/www/apphead/current/build_admin", "fs": "", "request_path": "/assets/img/support-open.svg", "result": "/var/www/apphead/current/build_admin/assets/img/support-open.svg"}
2024/10/09 15:53:24.076 DEBUG http.handlers.file_server sanitized path join {"site_root": "/var/www/apphead/current/build_admin", "fs": "", "request_path": "/assets/css/vendors-_yarn___virtual___datadog-browser-rum-virtual-25a58865cf_0_cache_datadog-browser-rum--f223b3.LeadingReach.css", "result": "/var/www/apphead/current/build_admin/assets/css/vendors-_yarn___virtual___datadog-browser-rum-virtual-25a58865cf_0_cache_datadog-browser-rum--f223b3.LeadingReach.css"}
2024/10/09 15:53:24.076 DEBUG http.handlers.file_server opening file {"filename": "/var/www/apphead/current/build_admin/assets/img/support-open.svg"}
2024/10/09 15:53:24.076 DEBUG http.handlers.file_server sanitized path join {"site_root": "/var/www/apphead/current/build_admin", "fs": "", "request_path": "/assets/img/support-close.svg", "result": "/var/www/apphead/current/build_admin/assets/img/support-close.svg"}
2024/10/09 15:53:24.077 DEBUG http.handlers.file_server opening file {"filename": "/var/www/apphead/current/build_admin/assets/css/vendors-_yarn___virtual___datadog-browser-rum-virtual-25a58865cf_0_cache_datadog-browser-rum--f223b3.LeadingReach.css"}
2024/10/09 15:53:24.077 DEBUG http.handlers.file_server sanitized path join {"site_root": "/var/www/apphead/current/build_admin", "fs": "", "request_path": "/assets/css/styles.LeadingReach.css", "result": "/var/www/apphead/current/build_admin/assets/css/styles.LeadingReach.css"}
2024/10/09 15:53:24.081 DEBUG http.handlers.file_server opening file {"filename": "/var/www/apphead/current/build_admin/assets/img/support-close.svg"}
2024/10/09 15:53:24.082 DEBUG http.handlers.file_server opening file {"filename": "/var/www/apphead/current/build_admin/assets/css/styles.LeadingReach.css"}
2024/10/09 15:53:24.107 DEBUG http.handlers.file_server sanitized path join {"site_root": "/var/www/apphead/current/build_admin", "fs": "", "request_path": "/assets/img/support-chat.svg", "result": "/var/www/apphead/current/build_admin/assets/img/support-chat.svg"}
2024/10/09 15:53:24.107 DEBUG http.handlers.file_server sanitized path join {"site_root": "/var/www/apphead/current/build_admin", "fs": "", "request_path": "/assets/img/support-helpdesk.svg", "result": "/var/www/apphead/current/build_admin/assets/img/support-helpdesk.svg"}
2024/10/09 15:53:24.108 DEBUG http.handlers.file_server opening file {"filename": "/var/www/apphead/current/build_admin/assets/img/support-chat.svg"}
2024/10/09 15:53:24.108 DEBUG http.handlers.file_server opening file {"filename": "/var/www/apphead/current/build_admin/assets/img/support-helpdesk.svg"}
2024/10/09 15:53:24.147 DEBUG http.handlers.file_server sanitized path join {"site_root": "/var/www/apphead/current/build_admin", "fs": "", "request_path": "/assets/js-wp/leadingreach-styles.c1e0fc335b259e630130.js", "result": "/var/www/apphead/current/build_admin/assets/js-wp/leadingreach-styles.c1e0fc335b259e630130.js"}
2024/10/09 15:53:24.148 DEBUG http.handlers.file_server opening file {"filename": "/var/www/apphead/current/build_admin/assets/js-wp/leadingreach-styles.c1e0fc335b259e630130.js"}
2024/10/09 15:53:24.167 DEBUG http.handlers.file_server sanitized path join {"site_root": "/var/www/apphead/current/build_admin", "fs": "", "request_path": "/assets/js-wp/leadingreach-admin.e80520bcb3f21a07bb58.js", "result": "/var/www/apphead/current/build_admin/assets/js-wp/leadingreach-admin.e80520bcb3f21a07bb58.js"}
2024/10/09 15:53:24.168 DEBUG http.handlers.file_server opening file {"filename": "/var/www/apphead/current/build_admin/assets/js-wp/leadingreach-admin.e80520bcb3f21a07bb58.js"}
2024/10/09 15:53:24.176 DEBUG http.handlers.rewrite rewrote request {"request": {"remote_ip": "192.168.65.1", "remote_port": "64025", "client_ip": "192.168.65.1", "proto": "HTTP/2.0", "method": "GET", "host": "fake-aws.leadingreach.localhost:8083", "uri": "/proxy/login_validate", "headers": {"Sec-Ch-Ua-Platform": ["\"macOS\""], "User-Agent": ["Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36"], "Sec-Ch-Ua-Mobile": ["?0"], "Accept": ["*/*"], "Sec-Fetch-Site": ["same-origin"], "Accept-Encoding": ["gzip, deflate, br, zstd"], "Priority": ["u=1, i"], "Sec-Ch-Ua": ["\"Google Chrome\";v=\"129\", \"Not=A?Brand\";v=\"8\", \"Chromium\";v=\"129\""], "Sec-Fetch-Mode": ["cors"], "Sec-Fetch-Dest": ["empty"], "Referer": ["https://fake-aws.leadingreach.localhost:8083/"], "Accept-Language": ["en-US,en;q=0.9"]}, "tls": {"resumed": false, "version": 772, "cipher_suite": 4865, "proto": "h2", "server_name": "fake-aws.leadingreach.localhost"}}, "method": "GET", "uri": "/login_validate"}
2024/10/09 15:53:24.177 DEBUG http.handlers.reverse_proxy selected upstream {"dial": ":15001", "total_upstreams": 1}
2024/10/09 15:53:24.179 DEBUG http.handlers.file_server sanitized path join {"site_root": "/var/www/apphead/current/build_admin", "fs": "", "request_path": "/assets/js-wp/leadingreach-vendors-_yarn___virtual___datadog-browser-rum-virtual-25a58865cf_0_cache_datadog-browser-rum--f223b3.7c6e1407d13ff1b2e4cd.js", "result": "/var/www/apphead/current/build_admin/assets/js-wp/leadingreach-vendors-_yarn___virtual___datadog-browser-rum-virtual-25a58865cf_0_cache_datadog-browser-rum--f223b3.7c6e1407d13ff1b2e4cd.js"}
2024/10/09 15:53:24.180 DEBUG http.handlers.file_server opening file {"filename": "/var/www/apphead/current/build_admin/assets/js-wp/leadingreach-vendors-_yarn___virtual___datadog-browser-rum-virtual-25a58865cf_0_cache_datadog-browser-rum--f223b3.7c6e1407d13ff1b2e4cd.js"}
2024/10/09 15:53:24.191 DEBUG events event {"name": "tls_get_certificate", "id": "bd0957f0-b668-4c34-84b4-f379ab28932c", "origin": "tls", "data": {"client_hello":{"CipherSuites":[49195,49199,49196,49200,52393,52392,49161,49171,49162,49172,49170,4865,4866,4867],"ServerName":"","SupportedCurves":[29,23,24,25],"SupportedPoints":"AA==","SignatureSchemes":[2052,1027,2055,2053,2054,1025,1281,1537,1283,1539,513,515],"SupportedProtos":["h2","http/1.1"],"SupportedVersions":[772,771],"RemoteAddr":{"IP":"127.0.0.1","Port":51016,"Zone":""},"LocalAddr":{"IP":"127.0.0.1","Port":15001,"Zone":""}}}}
2024/10/09 15:53:24.191 DEBUG tls.handshake no matching certificates and no custom selection logic {"identifier": "127.0.0.1"}
2024/10/09 15:53:24.192 DEBUG tls.handshake no certificate matching TLS ClientHello {"remote_ip": "127.0.0.1", "remote_port": "51016", "server_name": "", "remote": "127.0.0.1:51016", "identifier": "127.0.0.1", "cipher_suites": [49195, 49199, 49196, 49200, 52393, 52392, 49161, 49171, 49162, 49172, 49170, 4865, 4866, 4867], "cert_cache_fill": 0.0001, "load_or_obtain_if_necessary": true, "on_demand": false}
2024/10/09 15:53:24.192 DEBUG http.stdlib http: TLS handshake error from 127.0.0.1:51016: no certificate available for '127.0.0.1'
2024/10/09 15:53:24.193 DEBUG http.handlers.reverse_proxy upstream roundtrip {"upstream": ":15001", "duration": 0.015875459, "request": {"remote_ip": "192.168.65.1", "remote_port": "64025", "client_ip": "192.168.65.1", "proto": "HTTP/2.0", "method": "GET", "host": "fake-aws.leadingreach.localhost:8083", "uri": "/login_validate", "headers": {"Sec-Ch-Ua-Platform": ["\"macOS\""], "User-Agent": ["Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36"], "Sec-Ch-Ua-Mobile": ["?0"], "Sec-Ch-Ua": ["\"Google Chrome\";v=\"129\", \"Not=A?Brand\";v=\"8\", \"Chromium\";v=\"129\""], "Sec-Fetch-Dest": ["empty"], "X-Forwarded-For": ["192.168.65.1"], "X-Forwarded-Proto": ["https"], "Priority": ["u=1, i"], "Sec-Fetch-Mode": ["cors"], "Referer": ["https://fake-aws.leadingreach.localhost:8083/"], "Accept-Encoding": ["gzip, deflate, br, zstd"], "Accept": ["*/*"], "Sec-Fetch-Site": ["same-origin"], "Accept-Language": ["en-US,en;q=0.9"], "X-Forwarded-Host": ["fake-aws.leadingreach.localhost:8083"]}, "tls": {"resumed": false, "version": 772, "cipher_suite": 4865, "proto": "h2", "server_name": "fake-aws.leadingreach.localhost"}}, "error": "remote error: tls: internal error"}
2024/10/09 15:53:24.193 ERROR http.log.error remote error: tls: internal error {"request": {"remote_ip": "192.168.65.1", "remote_port": "64025", "client_ip": "192.168.65.1", "proto": "HTTP/2.0", "method": "GET", "host": "fake-aws.leadingreach.localhost:8083", "uri": "/proxy/login_validate", "headers": {"Priority": ["u=1, i"], "Sec-Ch-Ua-Platform": ["\"macOS\""], "User-Agent": ["Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36"], "Sec-Ch-Ua-Mobile": ["?0"], "Accept": ["*/*"], "Sec-Fetch-Site": ["same-origin"], "Accept-Encoding": ["gzip, deflate, br, zstd"], "Sec-Ch-Ua": ["\"Google Chrome\";v=\"129\", \"Not=A?Brand\";v=\"8\", \"Chromium\";v=\"129\""], "Sec-Fetch-Mode": ["cors"], "Sec-Fetch-Dest": ["empty"], "Referer": ["https://fake-aws.leadingreach.localhost:8083/"], "Accept-Language": ["en-US,en;q=0.9"]}, "tls": {"resumed": false, "version": 772, "cipher_suite": 4865, "proto": "h2", "server_name": "fake-aws.leadingreach.localhost"}}, "duration": 0.017667125, "status": 502, "err_id": "qjp4gu1tz", "err_trace": "reverseproxy.statusError (reverseproxy.go:1269)"}
I’m able to curl to my domain/port successfully, the issue seems to be specifically with the reverse_proxy? The errors below are coming from PHP, so it seems like the php_fastcgi is working as well.
root@094e336216ab:/var/www/apptail/current# curl https://fake-aws.leadingreach.localhost:15001/php-info
<!DOCTYPE html>
<html>
<head>
<meta charset="UTF-8" />
<meta http-equiv="refresh" content="0;url='/'" />
<title>Redirecting to /</title>
</head>
<body>
Redirecting to <a href="/">/</a>.
</body>
</html>
root@094e336216ab:/var/www/apptail/current# curl https://fake-aws.leadingreach.localhost:15001/privacy-policy
{"code":405,"message":"No route found for \u0022GET https:\/\/fake-aws.leadingreach.localhost:15001\/privacy-policy\u0022: Method Not Allowed (Allow: NONE, OPTIONS)"}
root@094e336216ab:/var/www/apptail/current#
3. Caddy version:
v2.8.4 h1:q3pe0wpBj1OcHFZ3n/1nl4V4bxBrYoSoab7rL9BMYNk=
4. How I installed and ran Caddy:
a. System environment:
Docker, it’s a debian-based container with shared services (don’t hate me, I was asked to do it this way for “simplicity” and “cost” reasons). Caddy is instaled
b. Command:
caddy start --config=/etc/caddy/Caddyfile
c. Service/unit/compose file:
Caddy installed in Docker image (Dockerfile):
RUN apt install -y debian-keyring debian-archive-keyring apt-transport-https curl
RUN curl -1sLf 'https://dl.cloudsmith.io/public/caddy/stable/gpg.key' | gpg --dearmor -o /usr/share/keyrings/caddy-stable-archive-keyring.gpg
RUN curl -1sLf 'https://dl.cloudsmith.io/public/caddy/stable/debian.deb.txt' | tee /etc/apt/sources.list.d/caddy-stable.list
RUN apt update
RUN apt install caddy
d. My complete Caddy config:
{
debug
log default {
format console
output file /var/log/caddy/system.log
exclude http.log.access
}
servers :8083 {
name appadmin
}
servers :15001 {
name api
}
}
fake-aws.leadingreach.localhost:8083 {
tls internal
handle_path /proxy/* {
reverse_proxy :15001 {
transport http {
tls_insecure_skip_verify
}
}
}
handle {
root * /var/www/apphead/current/build_admin
file_server
try_files {path} /index.html
}
}
##### Backend servers
fake-aws.leadingreach.localhost:15001, :15001 {
tls internal
root * /var/www/apptail/current/public
log {
output file /var/log/caddy/apptail-api.access.log
format console
}
# Encode responses in zstd or gzip, depending on the
# availability indicated by the browser.
encode zstd gzip
# Configures multiple PHP-related settings
php_fastcgi :14999
# Prevent access to dot-files, except .well-known
@dotFiles {
path */.*
not path /.well-known/*
}
}