1. Caddy version (caddy version):
2.4.5 built with Better fix for #142; patches 335551df · caddyserver/certmagic@88b8609 · GitHub
2. How I run Caddy:
a. System environment:
b. Command:
systemctl start caddy
c. Service/unit/compose file:
# caddy.service
#
# For using Caddy with a config file.
#
# Make sure the ExecStart and ExecReload commands are correct
# for your installation.
#
# See https://caddyserver.com/docs/install for instructions.
#
# WARNING: This service does not use the --resume flag, so if you
# use the API to make changes, they will be overwritten by the
# Caddyfile next time the service is restarted. If you intend to
# use Caddy's API to configure it, add the --resume flag to the
# `caddy run` command or use the caddy-api.service file instead.
# https://www.freedesktop.org/software/systemd/man/systemd.unit.html
[Unit]
Description=Caddy
Documentation=https://caddyserver.com/docs/
After=network.target network-online.target
Requires=network-online.target
# No more than StartLimitBurst service start during StartLimitIntervalSec period
StartLimitIntervalSec=500
StartLimitBurst=5
# https://www.freedesktop.org/software/systemd/man/systemd.service.html
[Service]
# Wait RestartSec before an auto restart on-failure
Restart=on-failure
RestartSec=5s
Type=notify
User=caddy
Group=caddy
ExecStart=/usr/bin/caddy run --environ --config /etc/caddy/Caddyfile
ExecReload=/usr/bin/caddy reload --config /etc/caddy/Caddyfile
TimeoutStopSec=5s
LimitNOFILE=1048576
LimitNPROC=512
PrivateTmp=true
ProtectSystem=full
AmbientCapabilities=CAP_NET_BIND_SERVICE
[Install]
WantedBy=multi-user.target
d. My complete Caddyfile or JSON config:
(log) {
log {
output file {args.0} {
roll_size 100MiB
}
level DEBUG
format json
format filter {
wrap json
fields {
common_log delete
resp_headers>Content-Security-Policy delete
request>remote_addr ip_mask {
ipv4 24
ipv6 32
}
}
}
}
}
{
debug
#Bind admin endpoint on all interfaces
admin 0.0.0.0:2019
on_demand_tls {
ask http://localhost:8000/api/cname/validate
}
# Workaround to prevent on_demand being enabled on all domains
# see https://github.com/caddyserver/caddy/pull/4128
email security+caddy@lempire.co
import log /var/log/caddy/default_req.log
# Reduce caddy reload grace_period to 3 min (default was 20 min) to improve reload
# See https://caddy.community/t/hangs-on-reload/12010/35
# Because of websockets long cnx, reload always took grace_period duration
grace_period 3m
}
# Redirect any accesses using an IP instead of a domain name
# hackerone report: https://hackerone.com/reports/723126
51.91.128.49:80,
91.121.37.140:80,
147.135.229.201:80,
51.254.243.65:80,
54.36.18.249:80,
46.105.54.36:80
{
redir https://app.lemlist.com{uri}
}
(cloudflare_matchers) {
remote_ip 103.21.244.0/22
remote_ip 103.22.200.0/22
remote_ip 103.31.4.0/22
remote_ip 104.16.0.0/13
remote_ip 104.24.0.0/14
remote_ip 108.162.192.0/18
remote_ip 131.0.72.0/22
remote_ip 141.101.64.0/18
remote_ip 162.158.0.0/15
remote_ip 172.64.0.0/13
remote_ip 173.245.48.0/20
remote_ip 188.114.96.0/20
remote_ip 190.93.240.0/20
remote_ip 197.234.240.0/22
remote_ip 198.41.128.0/17
remote_ip 2400:cb00::/32
remote_ip 2606:4700::/32
remote_ip 2803:f800::/32
remote_ip 2405:b500::/32
remote_ip 2405:8100::/32
remote_ip 2a06:98c0::/29
remote_ip 2c0f:f248::/32
}
(front_servers) {
to localhost:42001
to localhost:42002
to localhost:42003
to localhost:42004
to localhost:42005
to localhost:42006
to localhost:42007
to localhost:42008
health_uri /api/ping
health_interval 1s
health_timeout 5s
}
(badbots_filter) {
@badbots {
header_regexp User-Agent "(?i)Fuzz Faster U Fool.*"
}
header @badbots {
X-Cache-Debug "bbf"
Server ""
}
respond @badbots "Access denied" 403
}
#
# Main application
#
app.lemlist.com,
app2.lemlist.com
{
import log /var/log/caddy/front_req.log
# route keyword is mandatory to force Caddy to keep our
# directives order instead of using the Caddy builtin order
route {
request_body {
max_size 20MB
}
@from_cloudflare {
import cloudflare_matchers
}
@not_from_cloudflare {
not {
import cloudflare_matchers
}
}
##############################################
# pass all api lempire (e, i) requests to one
# dedicated Meteor app process
##############################################
@api_e_i {
path /api/lempire/*
}
header @api_e_i {
X-Cache-Debug "app.api_e_i"
Server ""
}
reverse_proxy @api_e_i {
to localhost:8000
}
##############################################
# pass all api requests to Meteor api process
##############################################
@api {
path /api/*
}
header @api {
X-Cache-Debug "app.api"
Server ""
}
reverse_proxy @api {
to localhost:42100
}
##############################################
# Caches
##############################################
# must be after /api routes or https://app.lemlist.com/api/froala/files/*.jpg will not be served by meteor
@cache_app {
path *.svg *.woff2 *.eot *.ttf *.woff *.jpg *.jpeg *.png *.gif *.ico *.html
}
header @cache_app {
# 1d expiration
Cache-Control max-age=86400
X-Cache-Debug "app.cache"
Server ""
}
file_server @cache_app {
root /opt/lemlist/app/programs/web.browser/app
}
@cache_app2 {
path_regexp ^/lib/.*\.(map|js|css)$
}
header @cache_app2 {
# 1d expiration
Cache-Control max-age=86400
X-Cache-Debug "app.cache2"
Server ""
}
file_server @cache_app2 {
root /opt/lemlist/app/programs/web.browser/app
}
@cache_version {
path_regexp ^/[0-9a-z]+\.(js|css|map).*
}
header @cache_version {
# 1d expiration
Cache-Control max-age=86400
X-Cache-Debug "app.cache_version"
Server ""
}
file_server @cache_version {
root /opt/lemlist/web.browser
}
##############################################
# Anything else goes to Meteor fronts
##############################################
header @from_cloudflare {
X-Cache-Debug "app.main.cf"
Server ""
}
reverse_proxy @from_cloudflare {
header_up X-Real-IP {http.request.header.CF-Connecting-IP}
header_up X-Forwarded-For {http.request.header.CF-Connecting-IP}
header_up Lb-Hash {http.request.header.CF-Connecting-IP}:{remote_port}
import front_servers
lb_policy header CF-Connecting-IP
}
header @not_from_cloudflare {
X-Cache-Debug "app.main"
Server ""
}
reverse_proxy @not_from_cloudflare {
header_up Lb-Hash {remote_host}:{remote_port}
import front_servers
lb_policy ip_hash
}
handle * {
respond 404
}
}
handle_errors {
@errors {
expression {http.error.status_code} == '502'
}
rewrite @errors /{http.error.status_code}.html
file_server {
root /var/www
}
}
#dev only : to work on laptop localhost
#tls internal
}
#
# Custom domains
#
(custom_domains) {
import log /var/log/caddy/custom_{args.0}_req.log
route {
import badbots_filter
#######################################
# render a special page when on root so the user can see if
# its custom domain is correctly setted up
#######################################
header / {
X-Cache-Debug "custom.cname_ok"
Server ""
}
file_server / {
root /var/www/lemlist_cname_ok
}
#######################################
# pass all requests "track open/click" to track
#######################################
@track {
path_regexp ^/api/(t/c|track/open|reply)/
}
header @track {
X-Cache-Debug "custom.track"
Server ""
}
reverse_proxy @track {
to localhost:42160
}
#######################################
# pass all requests "images + pages" to images
#######################################
@images {
path /*
}
header @images {
X-Cache-Debug "custom.images"
Server ""
}
reverse_proxy @images {
to localhost:42140
}
handle * {
respond 404
}
}
}
# WARNING : DO NOT add any other site address than :443 here
:443
{
import custom_domains on_demand
tls {
on_demand
issuer acme {
email security+caddy@lempire.co
dir https://acme.zerossl.com/v2/DV90
eab ...
...
}
issuer acme {
email security+caddy@lempire.co
dir https://acme-v02.api.letsencrypt.org/directory
}
}
}
pages.lemlist.com
{
import custom_domains predefined
}
# WARNING : DO NOT add any other site address than :80 here
:80
{
import custom_domains port80
}
#
# public direct access to service
#
(public_passthrough) {
{args.0}
{
import log /var/log/caddy/public_passthrough_{args.0}_req.log
route {
request_body {
max_size 20MB
}
header * {
X-Cache-Debug "{args.0}.main"
Server ""
}
reverse_proxy * {
to localhost:{args.1}
}
}
}
}
import public_passthrough api.lemlist.com 42100
import public_passthrough img.lemlist.com 42150
# https://front.lempire.co -> admin
import public_passthrough front.lem.ovh 8000
#
# private direct access to services
#
(private_passthrough) {
front.lem.ovh:{args.0}
{
import log /var/log/caddy/private_passthrough_{args.0}_req.log
route {
@cache_version {
path_regexp ^/[0-9a-z]+\.(js|css|map).*
}
header @cache_version {
# 1d expiration
Cache-Control max-age=86400
X-Cache-Debug "direct.cache_version"
Server ""
}
file_server @cache_version {
root /opt/lemlist/web.browser
}
header * {
X-Cache-Debug "direct.main"
Server ""
}
reverse_proxy * {
to localhost:{args.1}
}
}
basicauth * {
zzz ....
}
}
}
# Fronts
import private_passthrough 43001 42001
import private_passthrough 43002 42002
import private_passthrough 43003 42003
import private_passthrough 43004 42004
import private_passthrough 43005 42005
import private_passthrough 43006 42006
import private_passthrough 43007 42007
import private_passthrough 43008 42008
# API
import private_passthrough 43100 42100
# Cron
import private_passthrough 43130 42130
# Images Front
import private_passthrough 43150 42150
# Track
import private_passthrough 43160 42160
# Images
import private_passthrough 43140 42140
3. The problem I’m having:
Got 2 crashes in one week, not sure both are related… But I never got crashes with previous versions, using caddy for 6 months.
For the 1st crash, the service stopped.
Strangely the 2nd one was “only” a panic, the process was running, but sites were 408
May be let focus on 2nd one, as logs are interesting:
4. Error messages and/or full log output:
There is this very strange log line where removing_subjects is empty, same for removing_hash
"msg":"cache full; evicting random certificate","removing_subjects":[],"removing_hash":"","inserting_subjects":["pages.csmagency.ie"],"inserting_hash":"185532f43cf6f805e14cb7a035742456aadb0d6e400d578c8ecde2cfe1c090f4"
a bit more logs until the panic occured in removeCertificate():
{"level":"debug","ts":1633549994.4163284,"logger":"tls.handshake","msg":"matched certificate in cache","subjects":["custom.lemlist.com"],"managed":true,"expiration":1640822399,"hash":"a2fcedd5f125596efefa7c1021fa5d074e41d028876e084c4636ecd1da512122"}
2021-10-06 21:53:14
{"level":"debug","ts":1633549994.421294,"logger":"http.handlers.file_server","msg":"sanitized path join","site_root":"/var/www/lemlist_cname_ok","request_path":"/","result":"/var/www/lemlist_cname_ok"}
2021-10-06 21:53:14
{"level":"debug","ts":1633549994.4213588,"logger":"http.handlers.file_server","msg":"located index file","filename":"/var/www/lemlist_cname_ok/index.html"}
2021-10-06 21:53:14
{"level":"debug","ts":1633549994.4213848,"logger":"http.handlers.file_server","msg":"opening file","filename":"/var/www/lemlist_cname_ok/index.html"}
2021-10-06 21:53:14
{"level":"debug","ts":1633549994.4240758,"logger":"tls.handshake","msg":"choosing certificate","identifier":"trail.lemlist.com","num_choices":1}
2021-10-06 21:53:14
{"level":"debug","ts":1633549994.4241347,"logger":"tls.handshake","msg":"default certificate selection results","identifier":"trail.lemlist.com","subjects":["trail.lemlist.com"],"managed":true,"issuer_key":"acme.zerossl.com-v2-DV90","hash":"26c05f9b2e8d127a6fe949a1145302a843f44089c7567ae9e2c1ce1c8a854013"}
2021-10-06 21:53:14
{"level":"debug","ts":1633549994.4241555,"logger":"tls.handshake","msg":"matched certificate in cache","subjects":["trail.lemlist.com"],"managed":true,"expiration":1640822399,"hash":"26c05f9b2e8d127a6fe949a1145302a843f44089c7567ae9e2c1ce1c8a854013"}
2021-10-06 21:53:14
{"level":"debug","ts":1633549994.4276536,"logger":"tls.cache","msg":"cache full; evicting random certificate","removing_subjects":[],"removing_hash":"","inserting_subjects":["pages.csmagency.ie"],"inserting_hash":"185532f43cf6f805e14cb7a035742456aadb0d6e400d578c8ecde2cfe1c090f4"}
2021-10-06 21:53:14
{"level":"debug","ts":1633549994.4280682,"logger":"http.handlers.file_server","msg":"sanitized path join","site_root":"/var/www/lemlist_cname_ok","request_path":"/","result":"/var/www/lemlist_cname_ok"}
2021-10-06 21:53:14
{"level":"debug","ts":1633549994.4281065,"logger":"http.handlers.file_server","msg":"located index file","filename":"/var/www/lemlist_cname_ok/index.html"}
2021-10-06 21:53:14
{"level":"debug","ts":1633549994.4281197,"logger":"http.handlers.file_server","msg":"opening file","filename":"/var/www/lemlist_cname_ok/index.html"}
2021-10-06 21:53:14
{"level":"debug","ts":1633549994.4286802,"logger":"http.stdlib","msg":"http: panic serving 212.102.49.131:44993: runtime error: invalid memory address or nil pointer dereference\ngoroutine 71579125 [running]:\nnet/http.(*conn).serve.func1(0xc04fa35540)\n\t/usr/local/go/src/net/http/server.go:1824 +0x153\npanic(0x16d7ac0, 0x25baad0)\n\t/usr/local/go/src/runtime/panic.go:971 +0x499\ngithub.com/caddyserver/certmagic.(*Cache).removeCertificate(0xc0004808c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc0359e7e00, ...)\n\t/home/vagrant/go/pkg/mod/github.com/caddyserver/certmagic@v0.14.6-0.20210922202800-88b8609b4d8f/cache.go:281 +0x419\ngithub.com/caddyserver/certmagic.(*Cache).unsyncedCacheCertificate(0xc0004808c0, 0xc04e1d6180, 0x3, 0x4, 0x17d61c0, 0xc03321f800, 0x0, 0x0, 0x0, 0xc01f65ee00, ...)\n\t/home/vagrant/go/pkg/mod/github.com/caddyserver/certmagic@v0.14.6-0.20210922202800-88b8609b4d8f/cache.go:227 +0xbfd\ngithub.com/caddyserver/certmagic.(*Cache).cacheCertificate(0xc0004808c0, 0xc04e1d6180, 0x3, 0x4, 0x17d61c0, 0xc03321f800, 0x0, 0x0, 0x0, 0xc01f65ee00, ...)\n\t/home/vagrant/go/pkg/mod/github.com/caddyserver/certmagic@v0.14.6-0.20210922202800-88b8609b4d8f/cache.go:184 +0x98\ngithub.com/caddyserver/certmagic.(*Config).CacheManagedCertificate(0xc01570c820, 0xc047fa69a8, 0x12, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)\n\t/home/vagrant/go/pkg/mod/github.com/caddyserver/certmagic@v0.14.6-0.20210922202800-88b8609b4d8f/certificates.go:114 +0x1a5\ngithub.com/caddyserver/certmagic.(*Config).getCertDuringHandshake(0xc01570c820, 0xc008bea780, 0x300000101, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)\n\t/home/vagrant/go/pkg/mod/github.com/caddyserver/certmagic@v0.14.6-0.20210922202800-88b8609b4d8f/handshake.go:271 +0x17a6\ngithub.com/caddyserver/certmagic.(*Config).GetCertificate(0xc01570c820, 0xc008bea780, 0x12, 0xc000205600, 0x7f959a6d2878)\n\t/home/vagrant/go/pkg/mod/github.com/caddyserver/certmagic@v0.14.6-0.20210922202800-88b8609b4d8f/handshake.go:66 +0x118\ngithub.com/caddyserver/caddy/v2/modules/caddytls.(*ConnectionPolicy).buildStandardTLSConfig.func1(0xc008bea780, 0x410018, 0xb8, 0x1834fe0)\n\t/home/vagrant/caddy/modules/caddytls/connpolicy.go:185 +0x10f\ncrypto/tls.(*Config).getCertificate(0xc000701380, 0xc008bea780, 0xc008bea780, 0x18, 0x20)\n\t/usr/local/go/src/crypto/tls/common.go:1017 +0x3cb\ncrypto/tls.(*serverHandshakeState).processClientHello(0xc00e981968, 0xc000269380, 0x59a605)\n\t/usr/local/go/src/crypto/tls/handshake_server.go:223 +0x2c8\ncrypto/tls.(*serverHandshakeState).handshake(0xc00e981968, 0xc000b55400, 0x0)\n\t/usr/local/go/src/crypto/tls/handshake_server.go:64 +0x37\ncrypto/tls.(*Conn).serverHandshake(0xc00ab4f880, 0xc00299b6e0, 0x14)\n\t/usr/local/go/src/crypto/tls/handshake_server.go:58 +0x14b\ncrypto/tls.(*Conn).Handshake(0xc00ab4f880, 0x0, 0x0)\n\t/usr/local/go/src/crypto/tls/conn.go:1391 +0xc9\nnet/http.(*conn).serve(0xc04fa35540, 0x1bcc958, 0xc0008f5020)\n\t/usr/local/go/src/net/http/server.go:1840 +0x1a5\ncreated by net/http.(*Server).Serve\n\t/usr/local/go/src/net/http/server.go:3013 +0x39b"}
5. What I already tried:
I read carefully the certmagic/cache.go code, I cannot explain how is it possible to have an empty cert.Names
I grep “subject:”[]" in logs from start to the panic, the only occurence is the “evicting random certificate” line (I was expecting an occurence from a “added certificate to cache” log). So it seems no badly formed certificate have been pushed to the cache in 1st place.
May be it’s related to the fact that I built caddy myself …
6. Links to relevant resources:
How I built caddy on a fresh ubuntu 18.04 VM
sudo apt-get update
sudo apt install -y gcc
# Install go
wget https://golang.org/dl/go1.16.2.linux-amd64.tar.gz
sudo tar -C /usr/local -xzf go1.16.2.linux-amd64.tar.gz
export PATH=$PATH:/usr/local/go/bin
git clone https://github.com/caddyserver/caddy.git
cd caddy
go get -d -v github.com/caddyserver/certmagic@88b8609b4d8fb8c7b1865322602606b85240c80d
cd cmd/caddy/
go build