Caddy TSL Configuration over corporate load balancer

Help
1

1. The problem I’m having:

Can’t see to get SSL working. I know we have a corporate load balancer in place that routes all traffic but I’ve gotten this working before using Nginx with another application, tried using caddy and for some reason I can’t get it.

2. Error messages and/or full log output:

 {"level":"info","ts":1727889935.119009,"msg":"using config from file","file":"/etc/caddy/Caddyfile"}
 {"level":"info","ts":1727889935.1201584,"msg":"adapted config to JSON","adapter":"caddyfile"}
 {"level":"warn","ts":1727889935.1201708,"msg":"Caddyfile input is not formatted; run 'caddy fmt --overwrite' to fix inconsistencies","adapter":"caddyfile","file":"/etc/caddy/Caddyfile","line":13}
 {"level":"info","ts":1727889935.1225061,"logger":"admin","msg":"admin endpoint started","address":"localhost:2019","enforce_origin":false,"origins":["//localhost:2019","//[::1]:2019","//127.0.0.1:2019"]}
 {"level":"info","ts":1727889935.1227736,"logger":"tls.cache.maintenance","msg":"started background certificate maintenance","cache":"0xc000495400"}
 {"level":"debug","ts":1727889935.480209,"logger":"events","msg":"event","name":"cached_unmanaged_cert","id":"35963ac0-72e7-4ef3-9e37-9ef27cd7a75f","origin":"tls","data":{"sans":["*.nib-bahamas.com","nib-bahamas.com"]}}
 {"level":"debug","ts":1727889935.4802427,"logger":"tls.cache","msg":"added certificate to cache","subjects":["*.nib-bahamas.com","nib-bahamas.com"],"expiration":1756733222,"managed":false,"issuer_key":"","hash":"29e7a759d2715ecff3a7806cfd068f8d8ecfd1988d02dc7d07937e1c86c71e57","cache_size":1,"cache_capacity":10000}
 {"level":"info","ts":1727889935.4803548,"logger":"http.auto_https","msg":"automatic HTTPS is completely disabled for server","server_name":"srv0"}
 {"level":"info","ts":1727889935.4803622,"logger":"http.auto_https","msg":"automatic HTTPS is completely disabled for server","server_name":"srv1"}
 {"level":"debug","ts":1727889935.4803956,"logger":"http.auto_https","msg":"adjusted config","tls":{"automation":{"policies":[{}]}},"http":{"servers":{"srv0":{"listen":[":443"],"routes":[{"handle":[{"handler":"subroute","routes":[{"handle":[{"body":"Hello World","handler":"static_response"}]}]}],"terminal":true}],"tls_connection_policies":[{"match":{"sni":["staging-nibonline.nib-bahamas.com"]},"certificate_selection":{"any_tag":["cert0"]}},{}],"automatic_https":{"disable":true}},"srv1":{"listen":[":80"],"routes":[{"handle":[{"body":"Hello World","handler":"static_response"}]}],"automatic_https":{"disable":true}}}}}
 {"level":"info","ts":1727889935.4810925,"logger":"http","msg":"enabling HTTP/3 listener","addr":":443"}
 {"level":"info","ts":1727889935.4813006,"msg":"failed to sufficiently increase receive buffer size (was: 208 kiB, wanted: 7168 kiB, got: 416 kiB). See https://github.com/quic-go/quic-go/wiki/UDP-Buffer-Sizes for details."}
 {"level":"debug","ts":1727889935.4815354,"logger":"http","msg":"starting server loop","address":"[::]:443","tls":true,"http3":true}
 {"level":"info","ts":1727889935.4815483,"logger":"http.log","msg":"server running","name":"srv0","protocols":["h1","h2","h3"]}
 {"level":"debug","ts":1727889935.4815946,"logger":"http","msg":"starting server loop","address":"[::]:80","tls":false,"http3":false}
 {"level":"info","ts":1727889935.4815998,"logger":"http.log","msg":"server running","name":"srv1","protocols":["h1","h2","h3"]}
 {"level":"info","ts":1727889935.4818068,"msg":"autosaved config (load with --resume flag)","file":"/config/caddy/autosave.json"}
 {"level":"info","ts":1727889935.4818213,"msg":"serving initial configuration"}
 {"level":"info","ts":1727889935.4823184,"logger":"tls","msg":"cleaning storage unit","storage":"FileStorage:/data/caddy"}
 {"level":"info","ts":1727889935.4835443,"logger":"tls","msg":"finished cleaning storage units"}
 {"level":"debug","ts":1727889935.745238,"logger":"events","msg":"event","name":"tls_get_certificate","id":"bbc2bcbd-f671-42b1-9425-78b208d0a9ba","origin":"tls","data":{"client_hello":{"CipherSuites":[53,47,61,60,157,156,49172,49171,49192,49191,49200,49199,49162,49161,49188,49187,49196,49195,107,103,159,158,57,51,56,50],"ServerName":"","SupportedCurves":[23,24,21,25],"SupportedPoints":"AA==","SignatureSchemes":[1537,1538,1539,1281,1282,1283,1025,1026,1027,769,770,771,513,514,515],"SupportedProtos":null,"SupportedVersions":[771,770,769],"RemoteAddr":{"IP":"192.168.100.200","Port":48135,"Zone":""},"LocalAddr":{"IP":"172.24.0.2","Port":443,"Zone":""}}}}
 {"level":"debug","ts":1727889935.7452745,"logger":"tls.handshake","msg":"no matching certificates and no custom selection logic","identifier":"172.24.0.2"}
 {"level":"debug","ts":1727889935.745294,"logger":"tls.handshake","msg":"no certificate matching TLS ClientHello","remote_ip":"192.168.100.200","remote_port":"48135","server_name":"","remote":"192.168.100.200:48135","identifier":"172.24.0.2","cipher_suites":[53,47,61,60,157,156,49172,49171,49192,49191,49200,49199,49162,49161,49188,49187,49196,49195,107,103,159,158,57,51,56,50],"cert_cache_fill":0.0001,"load_or_obtain_if_necessary":true,"on_demand":false}
 {"level":"debug","ts":1727889935.7453773,"logger":"http.stdlib","msg":"http: TLS handshake error from 192.168.100.200:48135: no certificate available for '172.24.0.2'"}
 {"level":"debug","ts":1727889940.7443945,"logger":"events","msg":"event","name":"tls_get_certificate","id":"008bff5a-d4da-4dda-ac9c-c3c7594bd8c1","origin":"tls","data":{"client_hello":{"CipherSuites":[53,47,61,60,157,156,49172,49171,49192,49191,49200,49199,49162,49161,49188,49187,49196,49195,107,103,159,158,57,51,56,50],"ServerName":"","SupportedCurves":[23,24,21,25],"SupportedPoints":"AA==","SignatureSchemes":[1537,1538,1539,1281,1282,1283,1025,1026,1027,769,770,771,513,514,515],"SupportedProtos":null,"SupportedVersions":[771,770,769],"RemoteAddr":{"IP":"192.168.100.200","Port":25662,"Zone":""},"LocalAddr":{"IP":"172.24.0.2","Port":443,"Zone":""}}}}
 {"level":"debug","ts":1727889940.7444313,"logger":"tls.handshake","msg":"no matching certificates and no custom selection logic","identifier":"172.24.0.2"}
 {"level":"debug","ts":1727889940.7444468,"logger":"tls.handshake","msg":"no certificate matching TLS ClientHello","remote_ip":"192.168.100.200","remote_port":"25662","server_name":"","remote":"192.168.100.200:25662","identifier":"172.24.0.2","cipher_suites":[53,47,61,60,157,156,49172,49171,49192,49191,49200,49199,49162,49161,49188,49187,49196,49195,107,103,159,158,57,51,56,50],"cert_cache_fill":0.0001,"load_or_obtain_if_necessary":true,"on_demand":false}
 {"level":"debug","ts":1727889940.7445261,"logger":"http.stdlib","msg":"http: TLS handshake error from 192.168.100.200:25662: no certificate available for '172.24.0.2'"}

3. Caddy version:

v2.8.4

4. How I installed and ran Caddy:

a. System environment:

Using Docker with caddy. Nothing else. Just trying to get a response from my domain.

b. Command:

PASTE OVER THIS, BETWEEN THE ``` LINES.
Please use the preview pane to ensure it looks nice.

c. Service/unit/compose file:

services:
  webserver:
    image: caddy:alpine
    container_name: webserver
    restart: unless-stopped
    ports:
      - "80:80"
      - "8082:8082"
      - "443:443"
      - "443:443/udp"
    volumes:
      - ./caddy/Caddyfile:/etc/caddy/Caddyfile
      - caddy-data:/data
      - ./certs/star_nib_bahamas_com.pem:/etc/caddy/ssl/star_nib_bahamas_com.pem
      - ./certs/star_nib_bahamas_com.key:/etc/caddy/ssl/star_nib_bahamas_com.key
      - ./certs/star_nib_bahamas_com.crt:/etc/caddy/ssl/star_nib_bahamas_com.crt
    networks:
      - app-network

volumes:
  caddy-data:

networks:
  app-network:
    driver: bridge

d. My complete Caddy config:

{
	debug
	auto_https off
}

staging-nibonline.nib-bahamas.com:443 {
	tls /etc/caddy/ssl/star_nib_bahamas_com.pem /etc/caddy/ssl/star_nib_bahamas_com.key
	respond "Hello World"
}

:80 {
	respond "Hello World"
}
2

In this line it seems as if Caddy is looking for a certificate for the IP of the docker container, not the actual domain name provided. How can I fix this?

3

Does the LB send SNI for staging-nibonline.nib-bahamas.com?

If not, Caddy would be forced to assume a site address based on the IP address the LB is connecting to (which would be the Caddy container). Since it has no cert for the IP, it forces a HTTPS error rather than leaking certs of other configured sites.

You could resolve it by fixing the LB to signal SNI properly or telling Caddy what site to assume in cases of missing SNI: https://caddyserver.com/docs/caddyfile/options#default-sni

1 Like
4

Ah ok. That worked. This was a great help. The ultimate goal was actually get this to work with my wordpress docker container. Final working configuration to help anyone that needs it in the future.

compose.yml

services:
  db-sql:
    image: mysql:8.0
    restart: unless-stopped
    env_file: .env
    environment:
      - MYSQL_DATABASE=wordpress_new
    volumes:
      - dbnew_data:/var/lib/mysql
    command: '--default-authentication-plugin=mysql_native_password'
    networks:
      - app-network
    healthcheck:
      test: [ "CMD-SHELL", "exit | mysql -h localhost -P 3306 -u root -p$$MYSQL_ROOT_PASSWORD" ]

  wordpress:
    depends_on:
      db-sql:
        condition: service_healthy
    build: .
    restart: unless-stopped
    container_name: wordpress
    env_file: .env
    environment:
      - WORDPRESS_DB_HOST=db-sql:3306
      - WORDPRESS_DB_USER=$MYSQL_USER
      - WORDPRESS_DB_PASSWORD=$MYSQL_PASSWORD
      - WORDPRESS_DB_NAME=wordpress_new

    volumes:
      - wordpress_files:/var/www/html
    networks:
      - app-network

  webserver:
    depends_on:
      - wordpress
    image: caddy:alpine
    container_name: webserver
    restart: unless-stopped
    ports:
      - "80:80"
      - "443:443"
      - "443:443/udp"
    volumes:
      - ./caddy/Caddyfile:/etc/caddy/Caddyfile
      - caddy-data:/data
      - ./certs/domain_com.pem:/etc/caddy/ssl/domain_com.pem
      - ./certs/domain_com.key:/etc/caddy/ssl/domain_com.key
      - ./certs/domain_com.crt:/etc/caddy/ssl/domains_com.crt
    networks:
      - app-network

volumes:
  wordpress_files:
  dbnew_data:
  caddy-data:

networks:
  app-network:
    driver: bridge

Caddyfile

{
	debug
	auto_https off
	default_sni domain.com
}

http://domain.com {
	redir https://{host}{uri} permanent
}

https://domain.com {
	tls /etc/caddy/ssl/domain_com.pem /etc/caddy/ssl/domain_com.key
	reverse_proxy {
		to http://wordpress:80
	}
}
5

@Whitestrake Also if you have any suggestions on how to improve my config above I’d appreciate it! Very new to Caddy, only two days in. Loving it so far.

Thanks again.

6

Glad to see it’s working out. You could improve things a bit for sure.

auto_https off is superfluous here. It does two things - it turns off certificate management, and it turns off HTTP->S redirects. You’re already specifying a cert manually for your HTTPS site, so there’s no certificate to automate. You’ve also manually overridden the HTTP site already - although I’m not even sure that’s necessary! You’re just doing exactly what the automatic redirect does.

Obviously you also wanna probably turn off debug at some point unless you like the log verbosity.

Your reverse_proxy doesn’t need a subdirective block, and your upstream doesn’t need to specify the already-implicit scheme http:// or port :80, just wordpress will go to the right place.

Are you only serving one site? If so, you could make things even simpler still, removing the need to even specify default SNI.

Something as simple as this might have you completely covered:

https:// {
  tls /etc/caddy/ssl/domain_com.pem /etc/caddy/ssl/domain_com.key
  reverse_proxy wordpress
}
1 Like
7

This topic was automatically closed 30 days after the last reply. New replies are no longer allowed.