Caddy Won't Start After Update

Help
1

1. Caddy version (caddy version): v2.4.3 h1:Y1FaV2N4WO3rBqxSYA8UZsZTQdN+PwcoOcAiZTM8C0I=

2. How I run Caddy:

Personal webserver accessed over LAN, ZeroTier, and server.theundarkpixel.com

a. System environment: Ubuntu 20.04

b. Command:

(Via SystemD)

/usr/bin/caddy run --environ --config /etc/caddy/Caddyfile

c. Service/unit/compose file:

[Unit]
Description=Caddy
Documentation=https://caddyserver.com/docs/
After=network.target network-online.target
Requires=network-online.target

[Service]
Type=notify
User=caddy
Group=caddy
ExecStart=/usr/bin/caddy run --environ --config /etc/caddy/Caddyfile
ExecReload=/usr/bin/caddy reload --config /etc/caddy/Caddyfile
TimeoutStopSec=5s
LimitNOFILE=1048576
LimitNPROC=512
PrivateTmp=true
ProtectSystem=full
AmbientCapabilities=CAP_NET_BIND_SERVICE

[Install]
WantedBy=multi-user.target

d. My complete Caddyfile or JSON config:

(default) {
    root * /usr/share/caddy/public_html

    @php path /tt-rss/* /minecraft/solder/* /launcher-update/* #/rss-bridge/* /heimdall/*
    php_fastcgi @php unix//run/php/php7.4-fpm.sock

    #@heimdall_try {
    #    file {
    #        try_files /heimdall/public/* /heimdall/public/index.php
    #    }
    #}
    #rewrite @heimdall_try /heimdall/public/*
    #try_files /heimdall/* /heimdall/public/index.php
    #try_files /heimdall/public /heimdall/public/index.php
    #try_files /heimdall/public/* /heimdall/public/index.php

    file_server
}
(php) {
    php_fastcgi unix//run/php/php7.4-fpm.sock
}
(tt-rss) {
    root * /usr/share/caddy/tt-rss
    import php
    file_server
}
(heimdall) {
    root * /usr/share/caddy/heimdall/public
    import php
    file_server
}

server.theundarkpixel.com { # http://localhost, , http://192.168.1.34
    import heimdall
}

http://192.168.1.34 {
    tls internal
    #import default
    #handle_path /wallpapers/* {
    #    #root * /mnt/storage/aidan/Wallpapers/
    #    root * /usr/share/caddy/public_html
    #    import php
    #    file_server
    #}
    import heimdall
}

:6587 {
    root * /mnt/storage/aidan/Wallpapers
    import php
    #uri replace feed feed.php
    try_files {path} /index.php
    file_server #browse
}

ttrss.server.theundarkpixel.com { # , http://192.168.1.34
    import tt-rss
}

minecraftplus.server.theundarkpixel.com :7780 { # , http://192.168.1.34
    root * /usr/share/caddy/minecraftplus
    header *.wasm Content-Type application/wasm
    file_server
}

nextcloud.server.theundarkpixel.com :7867 {
    root * /usr/share/caddy/nextcloud
    file_server
    
    import php
    header {
        enable HSTS
        Strict-Transport-Security max-age=31536000;
    }

    #tls internal {
    #    on_demand
    #}

    redir /.well-known/carddav /remote.php/dav 301
    redir /.well-known/caldav /remote.php/dav 301

    # .htaccess / data / config / ... shouldn't be accessible from outside
    @forbidden {
        path    /.htaccess
        path    /data/*
        path    /config/*
        path    /db_structure
        path    /.xml
        path    /README
        path    /3rdparty/*
        path    /lib/*
        path    /templates/*
        path    /occ
        path    /console.php
    }

    respond @forbidden 404
}

http://192.168.1.34:8482 {
    tls internal
    import tt-rss
}

hass.server.theundarkpixel.com {
    reverse_proxy localhost:8123 {
        header_up Host {http.request.host}
        header_up X-Real-IP {http.request.remote}
        header_up X-Forwarded-For {http.request.remote}
        header_up X-Forwarded-Port {http.request.port}
        header_up X-Forwarded-Proto {http.request.scheme}
    }
}

plex.server.theundarkpixel.com {
    reverse_proxy localhost:32400
}

tautulli.server.theundarkpixel.com {
    reverse_proxy localhost:8181
}

friesandaioli.server.theundarkpixel.com {
    reverse_proxy localhost:2368
}

pterodactyl.server.theundarkpixel.com :8084 {
    root * /usr/share/caddy/pterodactyl/public

    header {
		X-Content-Type-Options nosniff
		X-XSS-Protection "1; mode=block"
		X-Robots-Tag none
		Content-Security-Policy "frame-ancestors 'self'"
		X-Frame-Options DENY
		Referrer-Policy same-origin
	}

	@htFiles {
		path_regexp /\.ht
	}
	respond @htFiles 403

    import php

	file_server
}

wings.server.theundarkpixel.com {
    reverse_proxy localhost:8087 {
        header_up Host {http.request.host}
        header_up X-Real-IP {http.request.remote}
        header_up X-Forwarded-For {http.request.remote}
        header_up X-Forwarded-Port {http.request.port}
        header_up X-Forwarded-Proto {http.request.scheme}
    }
}

:4945 {
    root * /usr/share/caddy/esivywp
    import php
    file_server browse
}

prometheus.theundarkpixel.com {
    reverse_proxy 192.168.1.70
}

#plex.prometheus.theundarkpixel.com {
#    reverse_proxy 192.168.1.70:32400
#}

tautulli.prometheus.theundarkpixel.com {
    reverse_proxy 192.168.1.70:8181
}

nextcloud.prometheus.theundarkpixel.com {
    reverse_proxy 192.168.1.70:7867
}

http://photoprism.prometheus.theundarkpixel.com {
    reverse_proxy 192.168.1.70:2342
}

#librephotos.server.theundarkpixel.com
#:7680 {
#    reverse_proxy :7680
#    reverse_proxy * :7682 {
#        header_up Host {http.request.host}
#    }
#    @api path /api/* /media/*
#    reverse_proxy @api :7681 {
#        header_up Host backend
#        header_up X-Real-IP {http.request.remote}
#    }
#    @protected_media {
#        remote_ip 127.0.0.1
#        path /protected_media*
#    }
#    file_server @protected_media {
#        root /var/lib/librephotos/
#    }
#    @original {
#        remote_ip 127.0.0.1
#        path /original*
#    }
#    handle @original {
#        uri strip_prefix /original
#        file_server {
#            root /var/lib/librephotos/data/
#        }
#    }
#    @nextcloud_original {
#        remote_ip 127.0.0.1
#        path /nextcloud_original*
#    }
#    handle @nextcloud_original {
#        uri strip_prefix /nextcloud_original
#        file_server {
#            root /var/lib/librephotos/data/nextcloud_media/
#        }
#    }
#}

#git {
#    path	launcher-update
#    repo	http://25.4.88.33:3000/Minecraft/Launcher.git
#    branch	update-site
#}

#git {
#    path	launcher-extras
#    repo	http://25.4.88.33:3000/Minecraft/Launcher.git
#    branch	extras
#}

#browse /launcher-extras

#fastcgi / /var/run/php/php7.0-fpm.sock {
#    ext 	.php
#    split	.php
#    #index	index.php
#}

# errors /home/caddy/errorlog.log

3. The problem I’m having:

Caddy just updated to v2.4.3 and now it won’t start with the below TLS error.

4. Error messages and/or full log output:

$ sudo -u caddy /usr/bin/caddy run --environ --config /etc/caddy/Caddyfile
caddy.HomeDir=/var/lib/caddy
caddy.AppDataDir=/var/lib/caddy/.local/share/caddy
caddy.AppConfigDir=/var/lib/caddy/.config/caddy
caddy.ConfigAutosavePath=/var/lib/caddy/.config/caddy/autosave.json
caddy.Version=v2.4.3 h1:Y1FaV2N4WO3rBqxSYA8UZsZTQdN+PwcoOcAiZTM8C0I=
runtime.GOOS=linux
runtime.GOARCH=amd64
runtime.Compiler=gc
runtime.NumCPU=8
runtime.GOMAXPROCS=8
runtime.Version=go1.16.5
os.Getwd=/home/aidan

LANG=en_US.UTF-8
LS_COLORS=rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=00:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.zst=01;31:*.tzst=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.wim=01;31:*.swm=01;31:*.dwm=01;31:*.esd=01;31:*.jpg=01;35:*.jpeg=01;35:*.mjpg=01;35:*.mjpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.webp=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=00;36:*.au=00;36:*.flac=00;36:*.m4a=00;36:*.mid=00;36:*.midi=00;36:*.mka=00;36:*.mp3=00;36:*.mpc=00;36:*.ogg=00;36:*.ra=00;36:*.wav=00;36:*.oga=00;36:*.opus=00;36:*.spx=00;36:*.xspf=00;36:
TERM=xterm-256color
PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/snap/bin
MAIL=/var/mail/caddy
LOGNAME=caddy
USER=caddy
HOME=/var/lib/caddy
SHELL=/usr/sbin/nologin
SUDO_COMMAND=/usr/bin/caddy run --environ --config /etc/caddy/Caddyfile
SUDO_USER=aidan
SUDO_UID=1000
SUDO_GID=1000
2021/06/22 01:10:42.007 INFO    using provided configuration    {"config_file": "/etc/caddy/Caddyfile", "config_adapter": ""}
2021/06/21 20:10:42 [WARNING] Unnecessary header_up ('X-Forwarded-Proto' field): the reverse proxy's default behavior is to pass headers to the upstream
2021/06/21 20:10:42 [WARNING] Unnecessary header_up ('X-Forwarded-Proto' field): the reverse proxy's default behavior is to pass headers to the upstream
2021/06/22 01:10:42.014 WARN    input is not formatted with 'caddy fmt' {"adapter": "caddyfile", "file": "/etc/caddy/Caddyfile", "line": 2}
2021/06/22 01:10:42.019 INFO    admin   admin endpoint started  {"address": "tcp/localhost:2019", "enforce_origin": false, "origins": ["localhost:2019", "[::1]:2019", "127.0.0.1:2019"]}
2021/06/22 01:10:42.020 INFO    tls.cache.maintenance   started background certificate maintenance      {"cache": "0xc0002e0d90"}
2021/06/22 01:10:42.030 INFO    tls.cache.maintenance   stopped background certificate maintenance      {"cache": "0xc0002e0d90"}
run: loading initial config: loading new config: loading tls app module: tls: invalid configuration: automation policy 2 is the second policy that acts as default/catch-all, but will never be used

5. What I already tried:

I tried to find any other services running that might interfere with TLS but I couldn’t find any. I had a similar problem where another machine on my network grabbed TLS credentials before this one but it’s unplugged right now.

6. Links to relevant resources:

2

enable is not a valid header. I think this was meant to be a comment, so put # in front of it.

Inline path matchers must start with a /. Use a named matcher for this. Otherwise, you end up with a weird header replacement handler which will never work.

Remove all of these lines. They are not useful, and in some cases harmful. Caddy sets the appropriate proxy headers automatically.

I think the problem is here:

You have an HTTP site with tls internal, which is a contradiction. This ends up generating an automation policy in the config which doesn’t make sense.

2 Likes
3

Hmm. That’s very true. I’m trying to decide if the current behavior is the right error… I’m guessing this happens because, to produce the list of subjects (domain names) associated with an automation policy, the non-HTTP hostnames are taken from the site block. In this case, there are none. Since it’s an empty list, but the automation policy is non-empty (since it contains a TLS configuration), the automation policy is created without subjects, thus becoming a catch-all.

Indeed this is a nonsensical config. But should we even make the automation policy in this case?

I will think on it, but since changing that code is gnarly – and gets worse every time – I want to just leave it as-is for now. I think an error like this might even be a good thing, since it reveals that an automation policy was created for a site block that doesn’t have any applicable hosts, which is definitely a red flag.

1 Like
4

That seems to have worked! On the proxy headers though- last time I tried to remove them Home Assistant freaked out. I’ll try it again soon though, no time tonight.

@matt

Indeed this is a nonsensical config.

Yeah, sorry about that. Not my field of expertise so I’m just trying to piece together what I can.

Thank you both so much for taking a look!

1 Like
5

No worries… it’s just a weird input and it’s not clear to me what the output should be (definitely should be an error, but, what kind of error? maybe what we have is good enough). Glad you got it working!

Thanks @francislavoie!

1 Like
6

This topic was automatically closed after 30 days. New replies are no longer allowed.