1. The problem I’m having:
Hi everyone, I’m trying to setup reverse proxy for immich with caddy and tailscale.
Tailscale is hosted on “host” network.
Caddy, portainer, immich, teslamate, grafana are hosted on a “bridge” network.
Facing issues with hosted applications on docker containers.
- If i remove tls_insecure_skip_verify from Caddyfile, it throws "400 Bad Request
The plain HTTP request was sent to HTTPS port".
Why is this happening only with grafana and immich? Portainer and teslamate are working fine without this. - After adding the tls_insecure_skip_verify, getting a blank page for immich and grafana with error in Caddy logs “status”:502,“err_trace”:"reverseproxy.statusError (reverseproxy.go:1267).
2. Error messages and/or full log output:
Grafana
{"level":"debug","ts":1712973737.5665598,"logger":"events","msg":"event","name":"tls_get_certificate","id":"ecc86f01-1eca-43cc-b5dc-23c5ec7a421c","origin":"tls","data":{"client_hello":{"CipherSuites":[4865,4867,4866,49195,49199,52393,52392,49196,49200,49162,49161,49171,49172,156,157,47,53],"ServerName":"synology-mytailscale.ts.net","SupportedCurves":[29,23,24,25,256,257],"SupportedPoints":"AA==","SignatureSchemes":[1027,1283,1539,2052,2053,2054,1025,1281,1537,515,513],"SupportedProtos":["h2","http/1.1"],"SupportedVersions":[772,771],"RemoteAddr":{"IP":"172.17.0.1","Port":58270,"Zone":""},"LocalAddr":{"IP":"172.17.0.12","Port":443,"Zone":""}}}}
{"level":"debug","ts":1712973737.5666752,"logger":"tls.handshake","msg":"no matching certificates and no custom selection logic","identifier":"synology-mytailscale.ts.net"}
{"level":"debug","ts":1712973737.5666854,"logger":"tls.handshake","msg":"no matching certificates and no custom selection logic","identifier":"*.chipmunk-tailor.ts.net"}
{"level":"debug","ts":1712973737.5666902,"logger":"tls.handshake","msg":"no matching certificates and no custom selection logic","identifier":"*.*.ts.net"}
{"level":"debug","ts":1712973737.5666947,"logger":"tls.handshake","msg":"no matching certificates and no custom selection logic","identifier":"*.*.*.net"}
{"level":"debug","ts":1712973737.566699,"logger":"tls.handshake","msg":"no matching certificates and no custom selection logic","identifier":"*.*.*.*"}
{"level":"debug","ts":1712973737.569988,"logger":"tls.handshake","msg":"using externally-managed certificate","remote_ip":"172.17.0.1","remote_port":"58270","sni":"synology-mytailscale.ts.net","names":["synology-mytailscale.ts.net"],"expiration":1720459124}
{"level":"debug","ts":1712973737.5781484,"logger":"http.handlers.rewrite","msg":"rewrote request","request":{"remote_ip":"172.17.0.1","remote_port":"58270","client_ip":"172.17.0.1","proto":"HTTP/2.0","method":"GET","host":"synology-mytailscale.ts.net","uri":"/grafana","headers":{"Accept-Encoding":["gzip, deflate, br"],"Sec-Fetch-Dest":["document"],"Sec-Fetch-Mode":["navigate"],"Sec-Fetch-Site":["none"],"Sec-Fetch-User":["?1"],"Accept":["text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8"],"Upgrade-Insecure-Requests":["1"],"Sec-Gpc":["1"],"Accept-Language":["en-US,en;q=0.5"],"Te":["trailers"],"User-Agent":["Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:124.0) Gecko/20100101 Firefox/124.0"],"Cookie":[],"Dnt":["1"]},"tls":{"resumed":false,"version":772,"cipher_suite":4865,"proto":"h2","server_name":"synology-mytailscale.ts.net"}},"method":"GET","uri":"/"}
{"level":"debug","ts":1712973737.5781956,"logger":"http.handlers.reverse_proxy","msg":"selected upstream","dial":"192.168.0.33:3000","total_upstreams":1}
{"level":"debug","ts":1712973737.5792289,"logger":"http.handlers.reverse_proxy","msg":"upstream roundtrip","upstream":"192.168.0.33:3000","duration":0.000963199,"request":{"remote_ip":"172.17.0.1","remote_port":"58270","client_ip":"172.17.0.1","proto":"HTTP/2.0","method":"GET","host":"synology-mytailscale.ts.net","uri":"/","headers":{"Sec-Fetch-Dest":["document"],"X-Forwarded-Proto":["https"],"X-Forwarded-Host":["synology-mytailscale.ts.net"],"User-Agent":["Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:124.0) Gecko/20100101 Firefox/124.0"],"Accept-Encoding":["gzip, deflate, br"],"Accept-Language":["en-US,en;q=0.5"],"Dnt":["1"],"Cookie":[],"Sec-Fetch-Mode":["navigate"],"Sec-Fetch-Site":["none"],"Accept":["text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8"],"Sec-Gpc":["1"],"Upgrade-Insecure-Requests":["1"],"X-Forwarded-For":["172.17.0.1"],"Sec-Fetch-User":["?1"],"Te":["trailers"]},"tls":{"resumed":false,"version":772,"cipher_suite":4865,"proto":"h2","server_name":"synology-mytailscale.ts.net"}},"error":"tls: first record does not look like a TLS handshake"}
{"level":"error","ts":1712973737.579313,"logger":"http.log.error","msg":"tls: first record does not look like a TLS handshake","request":{"remote_ip":"172.17.0.1","remote_port":"58270","client_ip":"172.17.0.1","proto":"HTTP/2.0","method":"GET","host":"synology-mytailscale.ts.net","uri":"/grafana","headers":{"Cookie":[],"Dnt":["1"],"Accept-Encoding":["gzip, deflate, br"],"Sec-Fetch-Dest":["document"],"Sec-Fetch-Mode":["navigate"],"Sec-Fetch-Site":["none"],"Sec-Fetch-User":["?1"],"Accept":["text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8"],"Upgrade-Insecure-Requests":["1"],"Sec-Gpc":["1"],"Accept-Language":["en-US,en;q=0.5"],"Te":["trailers"],"User-Agent":["Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:124.0) Gecko/20100101 Firefox/124.0"]},"tls":{"resumed":false,"version":772,"cipher_suite":4865,"proto":"h2","server_name":"synology-mytailscale.ts.net"}},"duration":0.001241482,"status":502,"err_id":"mt09dgcaz","err_trace":"reverseproxy.statusError (reverseproxy.go:1267)"}
immich
{"level":"debug","ts":1712973818.495334,"logger":"http.handlers.rewrite","msg":"rewrote request","request":{"remote_ip":"172.17.0.1","remote_port":"58270","client_ip":"172.17.0.1","proto":"HTTP/2.0","method":"GET","host":"synology-mytailscale.ts.net","uri":"/immich","headers":{"Cookie":[],"Te":["trailers"],"User-Agent":["Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:124.0) Gecko/20100101 Firefox/124.0"],"Sec-Fetch-Dest":["document"],"Sec-Fetch-Site":["none"],"Dnt":["1"],"Upgrade-Insecure-Requests":["1"],"Sec-Fetch-User":["?1"],"Sec-Gpc":["1"],"Accept-Language":["en-US,en;q=0.5"],"Accept-Encoding":["gzip, deflate, br"],"Sec-Fetch-Mode":["navigate"],"Accept":["text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8"]},"tls":{"resumed":false,"version":772,"cipher_suite":4865,"proto":"h2","server_name":"synology-mytailscale.ts.net"}},"method":"GET","uri":"/"}
{"level":"debug","ts":1712973818.4953842,"logger":"http.handlers.reverse_proxy","msg":"selected upstream","dial":"192.168.0.33:2283","total_upstreams":1}
{"level":"debug","ts":1712973818.4966915,"logger":"http.handlers.reverse_proxy","msg":"upstream roundtrip","upstream":"192.168.0.33:2283","duration":0.001243328,"request":{"remote_ip":"172.17.0.1","remote_port":"58270","client_ip":"172.17.0.1","proto":"HTTP/2.0","method":"GET","host":"synology-mytailscale.ts.net","uri":"/","headers":{"Upgrade-Insecure-Requests":["1"],"Sec-Fetch-Mode":["navigate"],"Sec-Fetch-Dest":["document"],"Sec-Gpc":["1"],"Sec-Fetch-User":["?1"],"Sec-Fetch-Site":["none"],"Accept-Language":["en-US,en;q=0.5"],"X-Forwarded-For":["172.17.0.1"],"X-Forwarded-Proto":["https"],"User-Agent":["Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:124.0) Gecko/20100101 Firefox/124.0"],"Te":["trailers"],"Dnt":["1"],"X-Forwarded-Host":["synology-mytailscale.ts.net"],"Cookie":[],"Accept":["text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8"],"Accept-Encoding":["gzip, deflate, br"]},"tls":{"resumed":false,"version":772,"cipher_suite":4865,"proto":"h2","server_name":"synology-mytailscale.ts.net"}},"error":"tls: first record does not look like a TLS handshake"}
{"level":"error","ts":1712973818.4967837,"logger":"http.log.error","msg":"tls: first record does not look like a TLS handshake","request":{"remote_ip":"172.17.0.1","remote_port":"58270","client_ip":"172.17.0.1","proto":"HTTP/2.0","method":"GET","host":"synology-mytailscale.ts.net","uri":"/immich","headers":{"Accept":["text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8"],"Accept-Language":["en-US,en;q=0.5"],"Accept-Encoding":["gzip, deflate, br"],"Sec-Fetch-Mode":["navigate"],"User-Agent":["Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:124.0) Gecko/20100101 Firefox/124.0"],"Cookie":[],"Te":["trailers"],"Dnt":["1"],"Sec-Fetch-Dest":["document"],"Sec-Fetch-Site":["none"],"Sec-Gpc":["1"],"Upgrade-Insecure-Requests":["1"],"Sec-Fetch-User":["?1"]},"tls":{"resumed":false,"version":772,"cipher_suite":4865,"proto":"h2","server_name":"synology-mytailscale.ts.net"}},"duration":0.001514853,"status":502,"err_id":"pi7i6rc30","err_trace":"reverseproxy.statusError (reverseproxy.go:1267)"}
3. Caddy version:
CADDY_VERSION v2.7.6
4. How I installed and ran Caddy:
Installed on Portainer with docker compose
a. System environment:
Synology DSM 7.2.1
Portainer - Docker version 20.10.23
c. My complete Caddy docker compose file:
version: "3.9"
services:
caddy:
image: caddy:latest
restart: unless-stopped
container_name: caddy
hostname: caddy
network_mode: bridge
ports:
- "80:80"
- "443:443"
- "443:443/udp"
volumes:
- /volume1/docker/caddy/Caddyfile:/etc/caddy/Caddyfile
- /volume1/docker/caddy/data:/data
- /volume1/docker/caddy/config:/config
# tailscale creates its socket on /tmp, so we'll kidnap from there to expose to caddy
- /volume1/docker/tailscale/tmp/tailscaled.sock:/var/run/tailscale/tailscaled.sock
d. My complete Caddy config:
{
debug
}
(network_paths) {
handle_path /portainer* {
reverse_proxy 192.168.0.1:9000
# Hitting "https://synology-mytailscale.ts.net/portainer"
# works fine, loads the portainer login page
}
handle_path /teslamate* {
reverse_proxy 192.168.0.1:4000. # teslamate is on port 4000:4000
# Hitting "https://synology.mytailscale.ts.net/teslamate"
# works fine, but its not rendering the teslamate webpage CSS properly
}
handle_path /grafana* {
reverse_proxy 192.168.0.1:3000 {. # grafana is on port 3000:3000
transport http {
tls_insecure_skip_verify
}
}
}
handle_path /immich* {
reverse_proxy 192.168.0.1:2283 { # Immich is on port 2283:3001
transport http {
tls_insecure_skip_verify
}
}
}
}
synology-mytailscale.ts.net {
import network_paths
}
e. Teslamate docker composer file
version: "3.9"
name: teslamate
services:
teslamate:
hostname: teslamate
image: teslamate/teslamate:latest
container_name: teslamate
restart: on-failure:5
environment:
- ENCRYPTION_KEY=fasldfjhweriuycxvzxdfwer
- DATABASE_USER=teslamate
- DATABASE_PASS=sdgljadhgweriuowvxvmn
- DATABASE_NAME=teslamate
- DATABASE_HOST=database
- MQTT_HOST=mosquitto
ports:
- 4000:4000
cap_drop:
- all
networks:
- bridge
database:
image: postgres:15
container_name: teslamate-db
restart: on-failure:5
environment:
- POSTGRES_USER=teslamate
- POSTGRES_PASSWORD=sdgljadhgweriuowvxvmn
- POSTGRES_DB=teslamate
volumes:
- /volume1/docker/teslamate/teslamate-db:/var/lib/postgresql/data:rw
networks:
- bridge
grafana:
image: teslamate/grafana:latest
container_name: teslamate-grafana
restart: on-failure:5
environment:
- DATABASE_USER=teslamate
- DATABASE_PASS=sdgljadhgweriuowvxvmn
- DATABASE_NAME=teslamate
- DATABASE_HOST=database
ports:
- 3000:3000
volumes:
- /volume1/docker/teslamate/teslamate-grafana:/var/lib/grafana:rw
networks:
- bridge
mosquitto:
image: eclipse-mosquitto:2
container_name: teslamate-mosquitto
restart: on-failure:5
command: mosquitto -c /mosquitto-no-auth.conf
# ports:
# - 1883:1883
volumes:
- /volume1/docker/teslamate/teslamate-mosquitto/config:/mosquitto/config:rw
- /volume1/docker/teslamate/teslamate-mosquitto/data:/mosquitto/data:rw
- /volume1/docker/teslamate/teslamate-mosquitto/log:/mosquitto/log:rw
networks:
- bridge
# Network definitions
networks:
# Define the network
bridge:
# Use the bridge driver
driver: bridge
f. Immich docker compose file
version: "3" # Use version 3 of the docker-compose file format
# Service definitions for the big-bear-immich application
services:
# Main Immich Server service configuration
immich-server:
hostname: immich
container_name: immich-server # Name of the running container
image: ghcr.io/immich-app/immich-server:v1.101.0 # Image to be used
command: ["start.sh", "immich"] # Command to be executed upon container start
ports: # Mapping ports from the host OS to the container
- 2283:3001
volumes: # Mounting directories for persistent data storage
- immich_upload:/usr/src/app/upload
- /volume1/homes/lashchdh/Photos:/usr/src/app/external:ro
environment: # Setting environment variables
DB_HOSTNAME: immich-postgres
DB_USERNAME: immich
DB_PASSWORD: ghjdhfhwrqewrxcvb
DB_DATABASE_NAME: immich
DB_PORT: 5432
REDIS_HOSTNAME: immich-redis
IMMICH_MACHINE_LEARNING_URL: http://immich-machine-learning:3003
depends_on: # Dependencies to ensure the order of service startup
- redis
- database
restart: always # Policy to always restart the container if it stops
networks:
- bridge
# Configuration for the Immich Microservices
immich-microservices:
container_name: immich-microservices # Name of the running container
image: ghcr.io/immich-app/immich-server:v1.101.0 # Image to be used
# extends: # Feature to use configuration from another service (currently commented out)
# file: hwaccel.yml
# service: hwaccel
command: ["start.sh", "microservices"] # Command to be executed upon container start
volumes: # Mounting directories for persistent data storage
- immich_upload:/usr/src/app/upload
- /volume1/homes/lashchdh/Photos:/usr/src/app/external:ro
environment: # Setting environment variables
DB_HOSTNAME: immich-postgres
DB_USERNAME: immich
DB_PASSWORD: ghjdhfhwrqewrxcvb
DB_DATABASE_NAME: immich
DB_PORT: 5432
REDIS_HOSTNAME: immich-redis
depends_on: # Dependencies to ensure the order of service startup
- redis
- database
restart: always # Policy to always restart the container if it stops
networks:
- bridge
# Configuration for Immich Machine Learning service
immich-machine-learning:
container_name: immich-machine-learning # Name of the running container
image: ghcr.io/immich-app/immich-machine-learning:v1.101.0 # Image to be used
volumes: # Mounting directories for persistent data storage
- immich_cache:/cache
environment: # Setting environment variables
DB_HOSTNAME: immich-postgres
DB_USERNAME: immich
DB_PASSWORD: ghjdhfhwrqewrxcvb
DB_DATABASE_NAME: immich
DB_PORT: 5432
REDIS_HOSTNAME: immich-redis
restart: always # Policy to always restart the container if it stops
networks:
- bridge
# Configuration for Redis service
redis:
container_name: immich-redis # Name of the running container
image: redis:6.2-alpine@sha256:70a7a5b641117670beae0d80658430853896b5ef269ccf00d1827427e3263fa3 # Image to be used
restart: always # Policy to always restart the container if it stops
networks:
- bridge
# Configuration for Database service
database:
container_name: immich-postgres # Name of the running container
image: tensorchord/pgvecto-rs:pg14-v0.2.0 # Image to be used
environment: # Setting environment variables
POSTGRES_PASSWORD: ghjdhfhwrqewrxcvb
POSTGRES_USER: immich
POSTGRES_DB: immich
PG_DATA: /var/lib/postgresql/data
volumes: # Mounting directories for persistent data storage
- immich_postgres:/var/lib/postgresql/data
restart: always # Policy to always restart the container if it stops
networks:
- bridge
# Network definitions
networks:
# Define the network
bridge:
# Use the bridge driver
driver: bridge
# Define named volumes for data persistence.
volumes:
# Define a named volume for Immich data.
immich_cache:
# Use the local storage driver.
driver: local
immich_upload:
# Use the local storage driver.
driver: local
immich_postgres:
# Use the local storage driver.
driver: local
5. What I already tried:
Search online for posts with similar errors on caddy forums, reddit and github, tried everything. I either get a blank page, 400 or 502 errors.
Not sure what to try beyond these.
