A few questions about caddyfile, converting from nginx config and using a service that provides its own certs

1. Caddy version (caddy version):

V2.4.6

2. How I run Caddy:

Docker

a. System environment:

skiffOS with docker

c. Service/unit/compose file:

version: '3.7'
services:

  nginx:
    depends_on:
      - radicale
      - homeassistant 
    image: nginx:latest
    container_name: nginx_reverse_proxy
    volumes:
      - /etc/localtime:/etc/localtime:ro
      - /mnt/persist/mydocker/nginx.conf:/etc/nginx/nginx.conf
      - /mnt/persist/mydocker/nginx/error.log:/etc/nginx/error.log
      - /mnt/persist/mydocker/nginx/access.log:/etc/nginx/access.log
      - /mnt/persist/mydocker/letsencrypt/:/etc/letsencrypt/
      - /mnt/persist/mydocker/nginx/dhparams.pem:/etc/nginx/dhparams.pem
      - snikket_data:/snikket:ro  
    restart: unless-stopped
    network_mode: host
 
  radicale:
    image: tomsquest/docker-radicale
    container_name: radicale
    ports:
      - 127.0.0.1:5232:5232
    init: true
    read_only: true
    security_opt:
      - no-new-privileges:true
    cap_drop:
      - ALL
    cap_add:
      - SETUID
      - SETGID
      - CHOWN
      - KILL
    healthcheck:
      test: curl -f http://127.0.0.1:5232 || exit 1
      interval: 30s
      retries: 3
    restart: unless-stopped
    volumes:
      - /etc/localtime:/etc/localtime:ro
      - /mnt/persist/mydocker/radicale/data:/data
      - /mnt/persist/mydocker/radicale/config:/config:ro
      - /mnt/persist/mydocker/radicale/users:/etc/radicale/users
      - /mnt/persist/mydocker/radicale/log:/var/log/radicale/log

  homeassistant:
    container_name: home-assistant
    image: homeassistant/home-assistant:stable
    volumes:
      - /etc/localtime:/etc/localtime:ro
      - /mnt/persist/mydocker/homeassistant:/config
    environment:
      - TZ=Europe/London
    restart: unless-stopped
    network_mode: host

  esphome:
    image: esphome/esphome
    volumes:
      - /mnt/persist/mydocker/esphome/config:/config:rw
      - /etc/localtime:/etc/localtime:ro
    network_mode: host
    restart: unless-stopped

  snikket_proxy:
    container_name: snikket-proxy
    image: snikket/snikket-web-proxy:dev
    env_file: snikket.conf
    network_mode: host
    volumes:
      - snikket_data:/snikket
      - acme_challenges:/var/www/html/.well-known/acme-challenge
    restart: "unless-stopped"
  snikket_certs:
    container_name: snikket-certs
    image: snikket/snikket-cert-manager:dev
    env_file: snikket.conf
    volumes:
      - snikket_data:/snikket
      - acme_challenges:/var/www/.well-known/acme-challenge
    restart: "unless-stopped"
  snikket_portal:
    container_name: snikket-portal
    image: snikket/snikket-web-portal:dev
    network_mode: host
    env_file: snikket.conf
    restart: "unless-stopped"

  snikket_server:
    container_name: snikket
    image: snikket/snikket-server:dev
    network_mode: host
    volumes:
      - snikket_data:/snikket
    env_file: snikket.conf
    restart: "unless-stopped"

volumes:
  acme_challenges:
  snikket_data:

Nginx config

user www-data;
worker_rlimit_core 500M;
worker_processes 1;

events {

  worker_connections 1024;

}

http {
  map $http_upgrade $connection_upgrade {
    default upgrade;
    ''      close;
  }
  error_log /etc/nginx/error.log warn;
  access_log /etc/nginx/access.log;
  ssl_dhparam /etc/nginx/dhparams.pem;
  ssl_prefer_server_ciphers on;
  ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384;
  ssl_protocols TLSv1.2 TLSv1.3;
  add_header Strict-Transport-Security "max-age=31536000; includeSubdomains";
  ssl_stapling on;
  ssl_stapling_verify on;
  ssl_certificate /etc/letsencrypt/live/myradicaleserver.duckdns.org/fullchain.pem;
  ssl_certificate_key /etc/letsencrypt/live/myradicaleserver.duckdns.org/privkey.pem;
  ssl_trusted_certificate /etc/letsencrypt/live/myradicaleserver.duckdns.org/chain.pem;
  ssl_session_cache shared:SSL:10m;
  proxy_buffering off;

  server {
    listen 80;
    server_name myradicaleserver.duckdns.org;
    return 301 https://$server_name$request_uri;    
  }
  
  server {
    listen 443 ssl http2;
    server_name myradicaleserver.duckdns.org;

    location /radicale/ {
    proxy_pass           http://localhost:5232/;
    proxy_set_header     X-Script-Name /radicale;
    proxy_set_header     X-Forwarded-For $proxy_add_x_forwarded_for;
    proxy_pass_header    Authorization;
    }
  }

  server {
    listen 80;
    server_name myhomeassistantserver.duckdns.org;
    return 301 https://$server_name$request_uri;
  }

  server {
    listen 443 ssl http2;
    server_name myhomeassistantserver.duckdns.org;

    location / {
        proxy_pass http://localhost:8123;
        proxy_set_header Host $host;
        proxy_http_version 1.1;
        proxy_set_header Upgrade $http_upgrade;
        proxy_set_header Connection "upgrade";
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
     }

     location /api/websocket {
        proxy_pass http://localhost:8123/api/websocket;
        proxy_set_header Host $host;
        proxy_http_version 1.1;
        proxy_set_header Upgrade $http_upgrade;
        proxy_set_header Connection "upgrade";
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;

     }
  }

  server {
   # Accept HTTP connections
    listen 80;
    listen [::]:80;

    server_name mysnikketserver.duckdns.org;
    server_name groups.mysnikketserver.duckdns.org;
    server_name share.mysnikketserver.duckdns.org;
    client_max_body_size 20M;
 
    location / {
        proxy_pass http://localhost:5080/;
        proxy_set_header      Host              $host;
        proxy_set_header      X-Forwarded-For   $proxy_add_x_forwarded_for;
     }
  }

  server {
    # Accept HTTPS connections
    listen [::]:443 ssl ipv6only=on;
    listen 443 ssl;
    ssl_certificate /snikket/letsencrypt/live/mysnikketserver.duckdns.org/fullchain.pem;
    ssl_certificate_key /snikket/letsencrypt/live/mysnikketserver.duckdns.org/privkey.pem;
    client_max_body_size 20M;       
 
    server_name mysnikketserver.duckdns.org;
    server_name groups.mysnikketserver.duckdns.org;
    server_name share.mysnikketserver.duckdns.org;

    location / {
        proxy_pass https://localhost:5443/;
        proxy_set_header      Host              $host;
        proxy_set_header      X-Forwarded-For   $proxy_add_x_forwarded_for;
        # REMOVE THIS IF YOU CHANGE `localhost` TO ANYTHING ELSE ABOVE
        proxy_ssl_verify      off;
        proxy_set_header      X-Forwarded-Proto https;
        proxy_ssl_server_name on;
     }
  }  
}

Hi

I could really use some advice currently I have a setup that uses nginx as a reverse proxy running in docker with several service ( home assistant , radicale, etc ) available remotely . I use a cron job to renew certs currently but I will be moving to a different operating system which is based on buildroot , SkiffOS .
so all service including cert renewal needs to run in containers , I did ask on the certbot forum about running certbot in a container and still using cron to renew the certs , but this seems a bit messy , why not take this time to switch to Caddy . link to certbot forum post

my questions are, I see that caddy does things like oscp stapling http to https redirects by default , but what else does it do by default , basically what can I remove from the above nginx config file thats not required in a caddyfile config . For example I know after a bit of research that I dont need the dhparam.pen or any of the cyphers (although I know there is an option for it )

I know a caddyfile is relatively simple to put together so something like home assistant can be as simple as

example.com {
         reverse_proxy localhost:8123
}

so I’m quite excited to get stuck in a create the caddyfile but do I need most of the global options from the nginx config for a secure caddy reverse proxy ?

my second question is I’ve seen mention of a duckdns plugin, is this something that ,I would need/benefit from as I use duckdns for my dns resolver , although I do have a static IP.

finally I have a xmpp server (snikket) running also in docker , It gets its own certs and I believe automatically proxys to https I know its possible to use external certs with caddy but I dont fully understand how.

thanks in advance for any advice / direction you have for me.

so far this is my first very basic attempt at understanding the caddyfile

{ 	   
log [access] {
	output  /var/log/access.log
	}		
}



myradicaleserver.duckdns.org{
	handle_path /radicale* {
		reverse_proxy localhost:5232 {
			header_up X-Script-Name /radicale
		}
	}
}

myhomeassistantserver.duckdns.org {
	reverse_proxy localhost:8123
}

http://mysnikketserver.duckdns.org, http://groups.mysnikketserver.duckdns.org, http://share.mysnikketserver.duckdns.org {
	reverse_proxy localhost:5080 localhost:5443
	request_body {
  	max_size 20MB
	}
}

Caddy’s defaults for TLS are secure, so you don’t need to tweak them. One of the major benefits of Caddy. Keeping the defaults means that upgrading Caddy will also upgrade your TLS settings should Caddy update them if new vulnerabilities are found.

Probably not.

It would mainly be useful if you need a wildcard certificate (DNS challenge is required by ACME CAs for wildcard certs), or if your sites are not publicly accessible (if it can’t be reached over ports 80/443 to solve the ACME HTTP and/or TLS-ALPN challenges), or if you want to use it alongside the dynamic_dns plugin (so that Caddy can update the IP on your behalf).

If you don’t need any of these, then you don’t need the duckdns plugin yet.

That’s trickier at this time. Getting a reliable path to the currently issued certificate in Caddy is not easy right now. I have some plans to make this easier eventually.

XMPP is tricky because it can either be used over HTTP, or raw TCP. Caddy is an HTTP server, so you could use it to terminate TLS and proxy XMPP, but only if you’re using XMPP over HTTP. That’ll be for you to figure out if you can make that work.

Caddy does have a plugin called caddy-l4 which can proxy TCP, but it currently doesn’t have Caddyfile support so it would be harder to use.

Careful here, you must have a space between the domain and the { for correct parsing.

Access logs won’t be enabled unless you use the log directive in each site. Generally, it’s best to use Caddyfile snippets to reuse the logging config in each site:

Any particular reason to use HTTP for these?

Using two upstreams will cause Caddy to load balance between the two. Are you sure this is what you want? They’re using different ports, making me think the second is the HTTPS of the upstream which would always fail to connect because your reverse_proxy isn’t configured to proxy over HTTPS. The reverse_proxy module defaults to proxying over HTTP, because Caddy typically has terminated TLS, so there’s rarely a reason to re-encrypt when sending to the upstream.

1 Like

Hi, thank you so much for your reply this has really helped,

this is kind of what I thought , which is why I would love to switch to caddy from nginx .

I definitely don’t need this thanks for the clarification .

I thought the xmpp part might be a bit difficult this is a PR for snikket for a caddy support guide here

at the top he mentions

To use Caddy with Snikket, you need to

  1. Forward HTTP traffic on port 80 with hostnames chat.example.com, groups.chat.example.com

and share.chat.example.com to Snikket’s port 5080.

  1. Forward HTTPS traffic on port 443 with the above SNIs to Snikket’s port 5443 without

terminating TLS, since Snikket obtains certificates by itself.

this is my feeble attempt at trying to turn off tls as mentioned above for snikket

and so is this

currently I provide the nginx reverse proxy with the certs from the snikket container under its server block , is this something I can achieve with caddy maybe ?

I will be using the logs for fail2ban mainly, and seeing as only home assistant provides logs at the moment its probably best I just stick this in there .

thanks again francislavoie

Proxying HTTPS without terminating TLS would require caddy-l4. It’s not possible to do with an HTTP server, because TLS bytes need to be decrypted for HTTP content to be read and handled.

I’m not sure why you’d want to serve HTTP at all. HTTP is not secure, obviously, so if you have HTTPS available, you should always use that.

It doesn’t make sense to proxy to both the HTTP and the HTTPS ports, you should just pick one. It’s definitely easier to proxy just to the HTTP port, simpler config.

If you do choose to proxy to the HTTPS port:

It depends on how the snikket certs were issued. Are they signed by a publicly trusted CA, or are they self-signed?

You need to at minimum enable tls for the upstream, which can be done by prefixing it with https://, but you’ll also need to configure trust if the upstream’s cert isn’t issued by a CA that’s trusted in the trust store of the system you have Caddy running on. In that case, you can either use the tls_trusted_ca_certs transport option to tell Caddy which cert to trust (recommended), or turn off verification altogether with tls_insecure_skip_verify (bad, drops all security, essentially the same as HTTP because anyone who gets between your Caddy server and your upstream can perform a man-in-the-middle attack).

so How do I do this , do I have to wait until an update to caddy ?

So As I understand it one of the snikket containers is a certbot container for creating its own lets encrypt
certs and another is a proxy container , this I think make the connection https to the snikket server container . Its meant to be an all in one xmpp server type thing . So I believe all it needs the reverse proxy server to do is redirect 80 to 5080 and 443 to 5443 and then it creates its own secure connection.

so maybe this is how I should achieve it I take it this goes in the site section of caddyfile ?
something like this ?

http://mysnikketserver.duckdns.org, http://groups.mysnikketserver.duckdns.org, http://share.mysnikketserver.duckdns.org {
	reverse_proxy localhost:5080
	request_body {
  	max_size 20MB
	}

mysnikketserver.duckdns.org, groups.mysnikketserver.duckdns.org, share.mysnikketserver.duckdns.org {
	reverse_proxy localhost:5443
        tls_insecure_skip_verify
	request_body {
  	max_size 20MB
	}

also for the logging I notice that you mentioned elsewhere about snippets so i can use

(log_common) {
  log {
    output file /var/log/caddy/{args.0}.access.log
  }
}

myradicaleserver.duckdns.org {
        handle_path /radicale* {
		reverse_proxy localhost:5232 {
			header_up X-Script-Name /radicale
                 }
	}
        import log_common myradicaleserver.duckdns.org
}
myhomeassistantserver.duckdns.org {
	reverse_proxy localhost:8123
        import log_common myhomeassistantserver.duckdns.org
}

  import log_common b.example.com

does log_common become a global snippet ? if so do I need curly braces over the entire entry ?

Hi! I’m a Snikket developer. I’m not (yet?) a Caddy user, but we’ve had a number of people recently struggling to get Caddy set up correctly with Snikket, and I’d love to be able to help them better.

This is because Snikket obtains certificates via ACME, and uses HTTP challenges. Unlike a web-only service, Snikket also requires certificates for XMPP (messaging) and TURN (audio/video relay) services, and these are generally not proxied.

It’s not a problem for a reverse proxy in front of Snikket’s HTTP/HTTPS ports to also do ACME, as long as ACME challenges for Snikket’s own certificate requests continue to get forwarded/served correctly.

Assuming Snikket can obtain its own certificates, and users don’t want to run multiple XMPP/TURN services (rare) on the same server, TLS passthrough and SNI sniffing should be unnecessary. You can simply terminate TLS for HTTPS and proxy it as normal.

Does this make sense? Happy to clarify anything if needed!

2 Likes

The plugin is not part of “vanilla” Caddy and needs to be added manually. You can add it by visiting this link, or add it to your Caddy build by using xcaddy from here.

xcaddy build --with github.com/mholt/caddy-l4

However, the plugin does not support the Caddyfile syntax, as @francislavoie said. In this case, I would recommend first figuring out the Caddyfile that works for you without Snikket. You can then convert it to JSON (or YAML), and follow the instructions provided in the PR to add Snikket.

caddy fmt --overwrite Caddyfile
caddy adapt --config Caddyfile --pretty > caddyConfig.json
2 Likes

how do I build this if I want to run in docker ? is this possible ? I should probably mention that this also runs on an arm based hardware so the docker would have to been arm also , correct ?

For docker, you can use the following Dockerfile. The image should work on arm as far as I can tell.

FROM caddy:builder AS builder

RUN xcaddy build \
    --with github.com/mholt/caddy-l4

FROM caddy:latest

COPY --from=builder /usr/bin/caddy /usr/bin/caddy

CMD ["caddy", "run", "--config", "/etc/caddy/config.json"]

If you are using Caddyfile, the last line should be omitted. In either case, the configuration file (Caddyfile / JSON) needs to be in /etc/caddy inside the container.

2 Likes

thankyou for this , I’m thinking now that even though the caddyfile is convenient , maybe I should try and write the whole thing in JSON that way I not converting my caddyfile to JSON and converting your PR from yaml to JSON also.

once I get a few days off work ,to experiment I’ll try and build a container with caddy-l4

You can use the YAML adapter directly as well. I personally find it easier to read.

FROM caddy:builder AS builder

RUN xcaddy build \
    --with github.com/mholt/caddy-l4 \
    --with github.com/abiosoft/caddy-yaml

FROM caddy:latest

COPY --from=builder /usr/bin/caddy /usr/bin/caddy

CMD ["caddy", "run", "--config", "/etc/caddy/config.yaml", "--adapter", "yaml"]

Thanks, I will add the things that have been discussed in this thread to the PR, and your feedback as well once you are done testing.

2 Likes

Thanks for helping out @Rijul-A :grinning_face_with_smiling_eyes:

:wave:

Welcome!

I think they should, but HTTP->HTTPS redirects would have to be disabled, because otherwise Caddy would respond with a redirect on HTTP requests that aren’t to a ACME HTTP challenge that Caddy can handle. So it might need to be like this:

http://mysnikketserver.duckdns.org,
http://groups.mysnikketserver.duckdns.org,
http://share.mysnikketserver.duckdns.org {
	reverse_proxy localhost:5080
}

mysnikketserver.duckdns.org,
groups.mysnikketserver.duckdns.org,
share.mysnikketserver.duckdns.org {
	reverse_proxy localhost:5443
}

So HTTP requests will still make it through to the upstream, no HTTP->HTTPS redirects. If Snikket performs redirects when appropriate, this should probably be fine I guess.

2 Likes

Thanks for the config snippet! I’ve finally got around to trying out Caddy to my test server, and with a couple of tweaks I’ve got it working:

http://example.com,
http://groups.example.com,
http://share.example.com {
	reverse_proxy localhost:5080
}

example.com,
groups.example.com,
share.example.com {
	reverse_proxy https://localhost:5443 {
		transport http {
			tls_insecure_skip_verify
		}
	}
}

The docs recommend against tls_insecure_skip_verify (for obvious reasons). It’s not terrible on localhost, but I did try tls_server_name {http.request.host}. However it seemed not to interpret the placeholder (I got the error certificate is valid for groups.example.com, share.example.com, example.com, not {http.request.host}). In any case I don’t know if this only changes SNI, or also the hostname that Caddy will verify the certificate against. Suggestions welcome!

3 Likes

I can also confirm that @MattJ solution seems to work I even added home assistant and radicale to the caddyfile and they seemed to work , unfortunately I didn’t notice till it was too late that the caddy_data volume wasn’t mounted correctly so the certs didn’t persist , now I have the lets encrypt rate limit :roll_eyes:

oh well lesson learned

I haven’t tested the logging config I mentioned yet, ill have to wait until I’m free from the rate limit .

Ah, bummer.

Should still be fine though because Caddy will fallback to ZeroSSL if it can’t get a cert from Let’s Encrypt. :+1:

Glad this ended up working out for you guys! :grin:

this is good to know thanks.

one issue I still have is all though it worked , caddy was still getting certs for snikket , in nginx

I specify in the snikket server block

ssl_certificate /snikket/letsencrypt/live/mysnikketserver.duckdns.org/fullchain.pem;
ssl_certificate_key /snikket/letsencrypt/live/mysnikketserver.duckdns.org/privkey.pem;

and of course I share the snikket data volume with nginx with

nginx:
    image: nginx:latest
    container_name: nginx_reverse_proxy
    volumes:
        - snikket_data:/snikket:ro

snikket_certs:
    container_name: snikket-certs
    image: snikket/snikket-cert-manager:dev
    volumes:
      - snikket_data:/snikket

volumes:
  acme_challenges:
  snikket_data:

is there a way to specify this in the caddyfile to use the certs that snikket already has ?

also this is my caddyfile to date , does anything look wrong?

(log_common) {
  log {
    output file /var/log/caddy/{args.0}.access.log
  }
}

myradicaleserver.duckdns.org {
	handle_path /radicale* {
		reverse_proxy localhost:5232 {
			header_up X-Script-Name /radicale
		}
	}
	import log_common myradicaleserver.duckdns.org
}

myhomeassistantserver.duckdns.org {
	reverse_proxy localhost:8123
	import log_common myhomeassistantserver.duckdns.org
}




http://mysnikketserver.duckdns.org,
http://groups.mysnikketserver.duckdns.org,
http://share.mysnikketserver.duckdns.org {
	reverse_proxy localhost:5080
	request_body {
  	max_size 20M
  	}
}

mysnikketserver.duckdns.org,
groups.mysnikketserver.duckdns.org,
share.mysnikketserver.duckdns.org {
	reverse_proxy https://localhost:5443 {
	    transport http {
                tls_insecure_skip_verify
            }
        }
        request_body {
  	    max_size 20M
  	}
}

Yes, with the tls directive, you can specify the cert and key to use.

Caddy’s ACME implementation is more robust though, FWIW, because it has multiple issuer support, OCSP stapling, rate limit avoidance, etc.

It’s only tricky right now to have a reliable path to the cert and key that Caddy manages because it can be in one of two directories because of issuer fallback.

so something like

http://mysnikketserver.duckdns.org,
http://groups.mysnikketserver.duckdns.org,
http://share.mysnikketserver.duckdns.org {
    tls /snikket/letsencrypt/live/mysnikketserver.duckdns.org/fullchain.pem /snikket/letsencrypt/live/mysnikketserver.duckdns.org/privkey.pem
	reverse_proxy localhost:5080
	request_body {
  	max_size 20MB
  	}
}

in both site blocks http and https ? or just http

I read the docs and took it as the path to the files, correct?

It doesn’t make sense to use tls in the HTTP site… Because it’s HTTP. It doesn’t use TLS. It would only make sense to use for the HTTPS site, if that’s what you’re going for.