1. Caddy version (caddy version):
Frontend Caddy reverse proxy server built with the Cloudflare module
root@caddy:~ # caddy version
v2.4.3 h1:Y1FaV2N4WO3rBqxSYA8UZsZTQdN+PwcoOcAiZTM8C0I=
Backend Caddy web server using a static binary
root@file:~ # caddy version
v2.4.3 h1:Y1FaV2N4WO3rBqxSYA8UZsZTQdN+PwcoOcAiZTM8C0I=
2. How I run Caddy:
a. System environment:
root@caddy:~ # freebsd-version
12.2-RELEASE-p6
b. Command:
service caddy start
c. Service/unit/compose file:
#!/bin/sh
# PROVIDE: caddy
# REQUIRE: LOGIN DAEMON NETWORKING
# KEYWORD: shutdown
# To enable caddy, add 'caddy_enable="YES"' to /etc/rc.conf or
# /etc/rc.conf.local
# Optional settings:
# caddy_config (string): Full path to caddy config file
# (/usr/local/etc/caddy/Caddyfile)
# caddy_adapter (string): Config adapter type (caddyfile)
# caddy_directory (string): Root for caddy storage (ACME certs, etc.)
# (/var/db/caddy)
# caddy_extra_flags (string): Extra flags passed to caddy start
# caddy_logdir (string): Where caddy logs are stored
# (/var/log/caddy)
# caddy_logfile (string): Location of process log (${caddy_logdir}/caddy.log)
# This is for startup/shutdown/error messages.
# To create an access log, see:
# https://caddyserver.com/docs/caddyfile/directives/log
# caddy_user (user): User to run caddy (root)
# caddy_group (group): Group to run caddy (wheel)
#
# This script will honor XDG_CONFIG_HOME/XDG_DATA_HOME. Caddy will create a
# .../caddy subdir in each of those. By default, they are subdirs of /var/db/caddy.
# See https://caddyserver.com/docs/conventions#data-directory
. /etc/rc.subr
name=caddy
rcvar=caddy_enable
desc="Powerful, enterprise-ready, open source web server with automatic HTTPS written in Go"
load_rc_config $name
# Defaults
: ${caddy_enable:=NO}
: ${caddy_adapter:=caddyfile}
: ${caddy_config:=/usr/local/etc/caddy/Caddyfile}
: ${caddy_directory:=/var/db/caddy}
: ${caddy_extra_flags:=""}
: ${caddy_logdir:="/var/log/${name}"}
: ${caddy_logfile:="${caddy_logdir}/${name}.log"}
: ${caddy_user:="root"}
: ${caddy_group:="wheel"}
# Config and base directories
: ${XDG_CONFIG_HOME:="${caddy_directory}/config"}
: ${XDG_DATA_HOME:="${caddy_directory}/data"}
export XDG_CONFIG_HOME XDG_DATA_HOME
command="/usr/local/bin/${name}"
caddy_flags="--config ${caddy_config} --adapter ${caddy_adapter}"
pidfile="/var/run/${name}/${name}.pid"
required_files="${caddy_config} ${command}"
start_precmd="caddy_precmd"
start_cmd="caddy_start"
stop_cmd="caddy_stop"
# Extra Commands
extra_commands="configtest reload"
configtest_cmd="caddy_command validate ${caddy_flags}"
reload_cmd="caddy_command reload ${caddy_flags}"
caddy_command()
{
/usr/bin/su -m "${caddy_user}" -c "${command} $*"
}
caddy_precmd()
{
# Create required directories and set permissions
/usr/bin/install -d -m 755 -o "${caddy_user}" -g "${caddy_group}" ${caddy_directory}
/usr/bin/install -d -m 700 -o "${caddy_user}" -g "${caddy_group}" ${caddy_directory}/config
/usr/bin/install -d -m 700 -o "${caddy_user}" -g "${caddy_group}" ${caddy_directory}/data
/usr/bin/install -d -m 755 -o "${caddy_user}" -g "${caddy_group}" ${caddy_logdir}
/usr/bin/install -d -m 700 -o "${caddy_user}" -g "${caddy_group}" /var/run/caddy
}
caddy_start()
{
echo -n "Starting caddy... "
/usr/bin/su -m ${caddy_user} -c "${command} start ${caddy_flags} \
${caddy_extra_flags} --pidfile ${pidfile}" >> ${caddy_logfile} 2>&1
if [ $? -eq 0 ] && ps -ax -o pid | grep -q "$(cat ${pidfile})"; then
echo "done"
echo "Log: ${caddy_logfile}"
else
echo "Error: Caddy failed to start"
echo "Check the caddy log: ${caddy_logfile}"
fi
}
caddy_stop()
{
echo -n "Stopping caddy... "
if caddy_command stop; then
echo "done"
else
echo "Error: Unable to stop caddy"
echo "Check the caddy log: ${caddy_logfile}"
fi
}
run_rc_command "$1"
d. My complete Caddyfile or JSON config:
This is based on the thread mTLS under FreeBSD. Only mTLS components are shown.
mTLS constructs for the frontend Caddyfile:
...
# Internal CA
acme.lan {
acme_server
tls internal
}
...
*.udance.com.au {
...
map {labels.3} {backend} {online} {mtls} {phpmyadmin} {
# HOSTNAME BACKEND ONLINE mTLS PHPMYADMIN #COMMENT
#---------------------------------------------------------------
...
file file.lan:443 yes yes no # file.udance.com.au
...
route {
...
# Secure backend communication
@mtls expression `{mtls} == "yes"`
reverse_proxy @mtls {backend} {
header_up Host {http.reverse_proxy.upstream.hostport}
header_up X-Forwarded-Host {host}
transport http {
tls
}
}
...
}
}
mTLS constructs for the backend Caddyfile:
{
...
acme_ca https://acme.lan/acme/local/directory
acme_ca_root /etc/ssl/certs/root.crt
}
file.lan {
root * /usr/local/www/caddy
file_server browse
}
3. The problem I’m having:
mTLS has broken 
seemingly since upgrading from Caddy 2.4.1 to 2.4.3.
4. Error messages and/or full log output:
Frontend process log extract:
{"level":"info","ts":1624176356.910208,"msg":"using provided configuration","config_file":"/usr/local/www/Caddyfile","config_adapter":"caddyfile"}
{"level":"warn","ts":1624176356.9237509,"msg":"input is not formatted with 'caddy fmt'","adapter":"caddyfile","file":"/usr/local/www/Caddyfile","line":2}
{"level":"info","ts":"2021-06-20T16:05:56.931+0800","logger":"admin","msg":"admin endpoint started","address":"tcp/localhost:2019","enforce_origin":false,"origins":["localhost:2019","[::1]:2019","127.0.0.1:2019"]}
{"level":"info","ts":"2021-06-20T16:05:56.931+0800","logger":"tls.cache.maintenance","msg":"started background certificate maintenance","cache":"0xc00036e070"}
{"level":"info","ts":"2021-06-20T16:05:56.953+0800","logger":"http","msg":"server is listening only on the HTTPS port but has no TLS connection policies; adding one to enable TLS","server_name":"srv0","https_port":443}
{"level":"info","ts":"2021-06-20T16:05:56.953+0800","logger":"http","msg":"enabling automatic HTTP->HTTPS redirects","server_name":"srv0"}
{"level":"info","ts":"2021-06-20T16:06:09.419+0800","logger":"pki.ca.local","msg":"root certificate is already trusted by system","path":"storage:pki/authorities/local/root.crt"}
{"level":"info","ts":"2021-06-20T16:06:09.419+0800","logger":"http","msg":"enabling automatic TLS certificate management","domains":["*.udance.com.au","caffigoalkeeping.com","www.xenografix.com.au","udance.com.au","acme.lan","caffigoalkeeping.com.au","readymcgetty.com.au","www.udance.com.au","www.caffigoalkeeping.com.au","www.readymcgetty.com.au","xenografix.com.au","www.caffigoalkeeping.com"]}
{"level":"info","ts":"2021-06-20T16:06:09.419+0800","logger":"tls","msg":"cleaning storage unit","description":"FileStorage:/var/db/caddy/data/caddy"}
{"level":"info","ts":"2021-06-20T16:06:09.439+0800","logger":"tls","msg":"finished cleaning storage units"}
{"level":"warn","ts":"2021-06-20T16:06:10.275+0800","logger":"tls","msg":"stapling OCSP","error":"no OCSP stapling for [acme.lan]: no OCSP server specified in certificate"}
{"level":"info","ts":"2021-06-20T16:06:10.275+0800","msg":"autosaved config (load with --resume flag)","file":"/var/db/caddy/config/caddy/autosave.json"}
{"level":"info","ts":"2021-06-20T16:06:10.275+0800","msg":"serving initial configuration"}
Successfully started Caddy (pid=34502) - Caddy is running in the background
2021/06/20 16:06:15 {"newNonce":"https://acme.lan/acme//local/new-nonce","newAccount":"https://acme.lan/acme//local/new-account","newOrder":"https://acme.lan/acme//local/new-order","revokeCert":"https://acme.lan/acme//local/revoke-cert","keyChange":"https://acme.lan/acme//local/key-change"}
{"level":"error","ts":"2021-06-20T16:08:59.307+0800","logger":"http.log.error.log4","msg":"remote error: tls: internal error","request":{"remote_addr":"10.1.1.222:60492","proto":"HTTP/2.0","method":"GET","host":"file.udance.com.au","uri":"/","headers":{"Sec-Fetch-Dest":["document"],"Accept-Language":["en-US,en;q=0.9"],"Upgrade-Insecure-Requests":["1"],"Accept":["text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9"],"Sec-Fetch-Site":["none"],"Sec-Fetch-Mode":["navigate"],"Sec-Fetch-User":["?1"],"Accept-Encoding":["gzip, deflate, br"],"Cache-Control":["max-age=0"],"Sec-Ch-Ua":["\" Not;A Brand\";v=\"99\", \"Microsoft Edge\";v=\"91\", \"Chromium\";v=\"91\""],"Sec-Ch-Ua-Mobile":["?0"],"User-Agent":["Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.101 Safari/537.36 Edg/91.0.864.48"]},"tls":{"resumed":false,"version":772,"cipher_suite":4865,"proto":"h2","proto_mutual":true,"server_name":"file.udance.com.au"}},"duration":0.003891467,"status":502,"err_id":"1k867wnxe","err_trace":"reverseproxy.statusError (reverseproxy.go:857)"}
Backend process log extract:
{"level":"info","ts":1624176375.1738176,"msg":"using provided configuration","config_file":"/usr/local/etc/caddy/Caddyfile","config_adapter":"caddyfile"}
{"level":"warn","ts":1624176375.1766913,"msg":"input is not formatted with 'caddy fmt'","adapter":"caddyfile","file":"/usr/local/etc/caddy/Caddyfile","line":12}
{"level":"info","ts":"2021-06-20T16:06:15.178+0800","logger":"admin","msg":"admin endpoint started","address":"tcp/localhost:2019","enforce_origin":false,"origins":["[::1]:2019","127.0.0.1:2019","localhost:2019"]}
{"level":"info","ts":"2021-06-20T16:06:15.179+0800","logger":"http","msg":"server is listening only on the HTTPS port but has no TLS connection policies; adding one to enable TLS","server_name":"srv0","https_port":443}
{"level":"info","ts":"2021-06-20T16:06:15.179+0800","logger":"http","msg":"enabling automatic HTTP->HTTPS redirects","server_name":"srv0"}
{"level":"info","ts":"2021-06-20T16:06:15.180+0800","logger":"http","msg":"enabling automatic TLS certificate management","domains":["file.lan"]}
{"level":"info","ts":"2021-06-20T16:06:15.181+0800","msg":"autosaved config (load with --resume flag)","file":"/var/db/caddy/config/caddy/autosave.json"}
{"level":"info","ts":"2021-06-20T16:06:15.181+0800","msg":"serving initial configuration"}
{"level":"info","ts":"2021-06-20T16:06:15.182+0800","logger":"tls.obtain","msg":"acquiring lock","identifier":"file.lan"}
Successfully started Caddy (pid=30213) - Caddy is running in the background
{"level":"info","ts":"2021-06-20T16:06:15.183+0800","logger":"tls.cache.maintenance","msg":"started background certificate maintenance","cache":"0xc00015aaf0"}
{"level":"info","ts":"2021-06-20T16:06:15.184+0800","logger":"tls","msg":"cleaning storage unit","description":"FileStorage:/var/db/caddy/data/caddy"}
{"level":"info","ts":"2021-06-20T16:06:15.184+0800","logger":"tls","msg":"finished cleaning storage units"}
{"level":"info","ts":"2021-06-20T16:06:15.207+0800","logger":"tls.obtain","msg":"lock acquired","identifier":"file.lan"}
{"level":"info","ts":"2021-06-20T16:06:15.231+0800","logger":"tls.issuance.acme","msg":"waiting on internal rate limiter","identifiers":["file.lan"],"ca":"https://acme.lan/acme/local/directory","account":""}
{"level":"info","ts":"2021-06-20T16:06:15.232+0800","logger":"tls.issuance.acme","msg":"done waiting on internal rate limiter","identifiers":["file.lan"],"ca":"https://acme.lan/acme/local/directory","account":""}
{"level":"error","ts":"2021-06-20T16:06:15.246+0800","logger":"tls.obtain","msg":"could not get certificate from issuer","identifier":"file.lan","issuer":"acme.lan-acme-local-directory","error":"[file.lan] creating new order: fetching new nonce from server: HTTP 404: (ca=https://acme.lan/acme/local/directory)"}
{"level":"error","ts":"2021-06-20T16:06:15.246+0800","logger":"tls.obtain","msg":"will retry","error":"[file.lan] Obtain: [file.lan] creating new order: fetching new nonce from server: HTTP 404: (ca=https://acme.lan/acme/local/directory)","attempt":1,"retrying_in":60,"elapsed":0.038701051,"max_duration":2592000}
{"level":"error","ts":"2021-06-20T16:09:15.293+0800","logger":"tls.obtain","msg":"could not get certificate from issuer","identifier":"file.lan","issuer":"acme.lan-acme-local-directory","error":"[file.lan] creating new order: fetching new nonce from server: HTTP 404: (ca=https://acme.lan/acme/local/directory)"}
{"level":"error","ts":"2021-06-20T16:09:15.293+0800","logger":"tls.obtain","msg":"will retry","error":"[file.lan] Obtain: [file.lan] creating new order: fetching new nonce from server: HTTP 404: (ca=https://acme.lan/acme/local/directory)","attempt":3,"retrying_in":120,"elapsed":180.085689977,"max_duration":2592000}
{"level":"error","ts":"2021-06-20T16:11:15.304+0800","logger":"tls.obtain","msg":"could not get certificate from issuer","identifier":"file.lan","issuer":"acme.lan-acme-local-directory","error":"[file.lan] creating new order: fetching new nonce from server: HTTP 404: (ca=https://acme.lan/acme/local/directory)"}
{"level":"error","ts":"2021-06-20T16:11:15.304+0800","logger":"tls.obtain","msg":"will retry","error":"[file.lan] Obtain: [file.lan] creating new order: fetching new nonce from server: HTTP 404: (ca=https://acme.lan/acme/local/directory)","attempt":4,"retrying_in":300,"elapsed":300.097287785,"max_duration":2592000}
{"level":"error","ts":"2021-06-20T16:16:15.315+0800","logger":"tls.obtain","msg":"could not get certificate from issuer","identifier":"file.lan","issuer":"acme.lan-acme-local-directory","error":"[file.lan] creating new order: fetching new nonce from server: HTTP 404: (ca=https://acme.lan/acme/local/directory)"}
{"level":"error","ts":"2021-06-20T16:16:15.315+0800","logger":"tls.obtain","msg":"will retry","error":"[file.lan] Obtain: [file.lan] creating new order: fetching new nonce from server: HTTP 404: (ca=https://acme.lan/acme/local/directory)","attempt":5,"retrying_in":600,"elapsed":600.108134007,"max_duration":2592000}
5. What I already tried:
Cross-referenced against post #65 in thread mTLS under FreeBSD
I tried deleting backend storage and restarting Caddy, but no change to the result. Similar error messages logged in the backend.
I’m back in business. Thanks