@francislavoie This is a follow-up to the trust store issue identified in the thread mTLS under FreeBSD and summarised specifically around the heading Limitations in post #65. In that, I described a semi-permanent workaround.
Within the SmallStep issue Not compatible with FreeBSD · Issue #1 · smallstep/truststore · GitHub, within this post, you mentioned the possibility of something more permanent, but it relied on three things coming together:
- certctl, which appeared for the first time under FreeBSD 12.0.
- A minimum of FreeBSD 12.2 - Up until 12.1, FreeBSD didn’t have a system-wide trust store. From 12.2, the trust store is located in
/etc/ssl/certs. - A newer version Go (v1.17) landing under FreeBSD.12.2.- Earlier versions of Go are not aware of the new FreeBSD trust store location.
Well, the planets have aligned and all these conditions now exist. I want to pick up from where we left off and then see where we go from here.
This extract from the certctl documentation
TRUSTPATH List of paths to search for trusted certificates. Default: _DESTDIR_/usr/share/certs/trusted _DESTDIR_/usr/local/share/certs _DESTDIR_/usr/local/etc/ssl/certs
Of these paths, the first two have content; the third path. it appears, doesn’t exist by default.
root@caddy:~ # ls /usr/share/certs/trusted
AC_RAIZ_FNMT-RCM.pem
ACCVRAIZ1.pem
Actalis_Authentication_Root_CA.pem
AffirmTrust_Commercial.pem
AffirmTrust_Networking.pem
AffirmTrust_Premium_ECC.pem
AffirmTrust_Premium.pem
Amazon_Root_CA_1.pem
Amazon_Root_CA_2.pem
Amazon_Root_CA_3.pem
Amazon_Root_CA_4.pem
Atos_TrustedRoot_2011.pem
Autoridad_de_Certificacion_Firmaprofesional_CIF_A62634068.pem
Baltimore_CyberTrust_Root.pem
Buypass_Class_2_Root_CA.pem
Buypass_Class_3_Root_CA.pem
CA_Disig_Root_R2.pem
Camerfirma_Chambers_of_Commerce_Root.pem
Camerfirma_Global_Chambersign_Root.pem
Certigna_Root_CA.pem
Certigna.pem
certSIGN_Root_CA_G2.pem
certSIGN_ROOT_CA.pem
Certum_Root_CA.pem
Certum_Trusted_Network_CA_2.pem
Certum_Trusted_Network_CA.pem
CFCA_EV_ROOT.pem
Chambers_of_Commerce_Root_-_2008.pem
Comodo_AAA_Services_root.pem
COMODO_Certification_Authority.pem
COMODO_ECC_Certification_Authority.pem
COMODO_RSA_Certification_Authority.pem
Cybertrust_Global_Root.pem
D-TRUST_Root_CA_3_2013.pem
D-TRUST_Root_Class_3_CA_2_2009.pem
D-TRUST_Root_Class_3_CA_2_EV_2009.pem
DigiCert_Assured_ID_Root_CA.pem
DigiCert_Assured_ID_Root_G2.pem
DigiCert_Assured_ID_Root_G3.pem
DigiCert_Global_Root_CA.pem
DigiCert_Global_Root_G2.pem
DigiCert_Global_Root_G3.pem
DigiCert_High_Assurance_EV_Root_CA.pem
DigiCert_Trusted_Root_G4.pem
DST_Root_CA_X3.pem
e-Szigno_Root_CA_2017.pem
E-Tugra_Certification_Authority.pem
EC-ACC.pem
emSign_ECC_Root_CA_-_C3.pem
emSign_ECC_Root_CA_-_G3.pem
emSign_Root_CA_-_C1.pem
emSign_Root_CA_-_G1.pem
Entrust_net_Premium_2048_Secure_Server_CA.pem
Entrust_Root_Certification_Authority_-_EC1.pem
Entrust_Root_Certification_Authority_-_G2.pem
Entrust_Root_Certification_Authority_-_G4.pem
Entrust_Root_Certification_Authority.pem
ePKI_Root_Certification_Authority.pem
GDCA_TrustAUTH_R5_ROOT.pem
GeoTrust_Primary_Certification_Authority_-_G2.pem
Global_Chambersign_Root_-_2008.pem
GlobalSign_ECC_Root_CA_-_R4.pem
GlobalSign_ECC_Root_CA_-_R5.pem
GlobalSign_Root_CA_-_R2.pem
GlobalSign_Root_CA_-_R3.pem
GlobalSign_Root_CA_-_R6.pem
GlobalSign_Root_CA.pem
Go_Daddy_Class_2_CA.pem
Go_Daddy_Root_Certificate_Authority_-_G2.pem
GTS_Root_R1.pem
GTS_Root_R2.pem
GTS_Root_R3.pem
GTS_Root_R4.pem
Hellenic_Academic_and_Research_Institutions_ECC_RootCA_2015.pem
Hellenic_Academic_and_Research_Institutions_RootCA_2011.pem
Hellenic_Academic_and_Research_Institutions_RootCA_2015.pem
Hongkong_Post_Root_CA_1.pem
Hongkong_Post_Root_CA_3.pem
IdenTrust_Commercial_Root_CA_1.pem
IdenTrust_Public_Sector_Root_CA_1.pem
ISRG_Root_X1.pem
Izenpe_com.pem
Microsec_e-Szigno_Root_CA_2009.pem
Microsoft_ECC_Root_Certificate_Authority_2017.pem
Microsoft_RSA_Root_Certificate_Authority_2017.pem
NAVER_Global_Root_Certification_Authority.pem
NetLock_Arany__Class_Gold__F__tan__s__tv__ny.pem
Network_Solutions_Certificate_Authority.pem
OISTE_WISeKey_Global_Root_GA_CA.pem
OISTE_WISeKey_Global_Root_GB_CA.pem
OISTE_WISeKey_Global_Root_GC_CA.pem
QuoVadis_Root_CA_1_G3.pem
QuoVadis_Root_CA_2_G3.pem
QuoVadis_Root_CA_2.pem
QuoVadis_Root_CA_3_G3.pem
QuoVadis_Root_CA_3.pem
QuoVadis_Root_CA.pem
Secure_Global_CA.pem
SecureSign_RootCA11.pem
SecureTrust_CA.pem
Security_Communication_Root_CA.pem
Security_Communication_RootCA2.pem
Sonera_Class_2_Root_CA.pem
SSL_com_EV_Root_Certification_Authority_ECC.pem
SSL_com_EV_Root_Certification_Authority_RSA_R2.pem
SSL_com_Root_Certification_Authority_ECC.pem
SSL_com_Root_Certification_Authority_RSA.pem
Staat_der_Nederlanden_EV_Root_CA.pem
Staat_der_Nederlanden_Root_CA_-_G3.pem
Starfield_Class_2_CA.pem
Starfield_Root_Certificate_Authority_-_G2.pem
Starfield_Services_Root_Certificate_Authority_-_G2.pem
SwissSign_Gold_CA_-_G2.pem
SwissSign_Platinum_CA_-_G2.pem
SwissSign_Silver_CA_-_G2.pem
Symantec_Class_1_Public_Primary_Certification_Authority_-_G6.pem
Symantec_Class_2_Public_Primary_Certification_Authority_-_G6.pem
SZAFIR_ROOT_CA2.pem
T-TeleSec_GlobalRoot_Class_2.pem
T-TeleSec_GlobalRoot_Class_3.pem
TeliaSonera_Root_CA_v1.pem
TrustCor_ECA-1.pem
TrustCor_RootCert_CA-1.pem
TrustCor_RootCert_CA-2.pem
Trustis_FPS_Root_CA.pem
Trustwave_Global_Certification_Authority.pem
Trustwave_Global_ECC_P256_Certification_Authority.pem
Trustwave_Global_ECC_P384_Certification_Authority.pem
TUBITAK_Kamu_SM_SSL_Kok_Sertifikasi_-_Surum_1.pem
TWCA_Global_Root_CA.pem
TWCA_Root_Certification_Authority.pem
UCA_Extended_Validation_Root.pem
UCA_Global_G2_Root.pem
USERTrust_ECC_Certification_Authority.pem
USERTrust_RSA_Certification_Authority.pem
Verisign_Class_1_Public_Primary_Certification_Authority_-_G3.pem
Verisign_Class_2_Public_Primary_Certification_Authority_-_G3.pem
VeriSign_Universal_Root_Certification_Authority.pem
XRamp_Global_CA_Root.pem
root@caddy:~ # ls /usr/local/share/certs
ca-root-nss.crt
The Caddy local CA root certificate is located in /var/db/caddy/data/caddy/pki/authorities/local/root.crt
It seemed to me that the steps I needed to now undertake are:
- Create that third path and reference the Caddy local CA certificate.
- Rebuild the list of trusted certificate authorities using
certctl rehash. Symbolic links to the trusted certificates are placed in the FreeBSD trust store located at/etc/ssl/certs. - Rebuild Caddy using Go 1.17 or later so it becomes aware of the FreeBSD trust store.
Step 1 - Reference the Caddy local certificate in a path that certctl will search
$ mkdir -p /usr/local/etc/ssl/certs
$ ln -s /var/db/caddy/data/caddy/pki/authorities/local/root.crt /usr/local/etc/ssl/certs/root.crt
Step 2 - Rebuild the trust store
$ certctl rehash
Scanning /usr/share/certs/blacklisted for certificates...
Scanning /usr/share/certs/trusted for certificates...
Scanning /usr/local/share/certs for certificates...
Scanning /usr/local/etc/ssl/certs for certificates...
To check that the Caddy local CA certificate is now in the trust store, I executed the following command:
$ certctl list | grep Caddy
1e027192.0 Caddy Local Authority - 2021 ECC Root
Who-hoo! This is a good sign.
Step 3 - Rebuild Caddy using Go 1.17 or later and restart the Caddy service.
$ pkg info | grep Go
go-1.17.1,1 Go programming language
$
$ xcaddy build master --output /usr/local/bin/caddy --with github.com/caddy-dns/cloudflare
2021/11/20 11:39:41 [INFO] Temporary folder: /tmp/buildenv_2021-11-20-1139.311727266
2021/11/20 11:39:41 [INFO] Writing main module: /tmp/buildenv_2021-11-20-1139.311727266/main.go
2021/11/20 11:39:41 [INFO] Initializing Go module
2021/11/20 11:39:41 [INFO] exec (timeout=10s): /usr/local/bin/go mod init caddy
go: creating new go.mod: module caddy
go: to add module requirements and sums:
go mod tidy
2021/11/20 11:39:41 [INFO] Pinning versions
2021/11/20 11:39:41 [INFO] exec (timeout=0s): /usr/local/bin/go get -d -v github.com/caddyserver/caddy/v2@master
go: downloading github.com/caddyserver/caddy/v2 v2.4.7-0.20211116200822-7f364c777acf
go get: added github.com/beorn7/perks v1.0.1
go get: added github.com/caddyserver/caddy/v2 v2.4.7-0.20211116200822-7f364c777acf
go get: added github.com/caddyserver/certmagic v0.15.2
go get: added github.com/cespare/xxhash/v2 v2.1.1
go get: added github.com/golang/protobuf v1.5.2
go get: added github.com/google/uuid v1.3.0
go get: added github.com/klauspost/cpuid/v2 v2.0.9
go get: added github.com/libdns/libdns v0.2.1
go get: added github.com/matttproud/golang_protobuf_extensions v1.0.1
go get: added github.com/mholt/acmez v1.0.1
go get: added github.com/miekg/dns v1.1.43
go get: added github.com/prometheus/client_golang v1.11.0
go get: added github.com/prometheus/client_model v0.2.0
go get: added github.com/prometheus/common v0.26.0
go get: added github.com/prometheus/procfs v0.6.0
go get: added go.uber.org/atomic v1.7.0
go get: added go.uber.org/multierr v1.6.0
go get: added go.uber.org/zap v1.19.0
go get: added golang.org/x/crypto v0.0.0-20210915214749-c084706c2272
go get: added golang.org/x/net v0.0.0-20210913180222-943fd674d43e
go get: added golang.org/x/sys v0.0.0-20210915083310-ed5796bab164
go get: added golang.org/x/term v0.0.0-20210503060354-a79de5458b56
go get: added golang.org/x/text v0.3.7
go get: added google.golang.org/protobuf v1.27.1
2021/11/20 11:40:02 [INFO] exec (timeout=0s): /usr/local/bin/go get -d -v github.com/caddy-dns/cloudflare
go get: added github.com/caddy-dns/cloudflare v0.0.0-20210607183747-91cf700356a1
go get: added github.com/libdns/cloudflare v0.1.0
2021/11/20 11:40:04 [INFO] Build environment ready
2021/11/20 11:40:04 [INFO] Building Caddy
2021/11/20 11:40:04 [INFO] exec (timeout=0s): /usr/local/bin/go mod tidy
2021/11/20 11:40:11 [INFO] exec (timeout=0s): /usr/local/bin/go build -o /usr/local/bin/caddy -ldflags -w -s -trimpath
2021/11/20 11:40:44 [INFO] Build complete: /usr/local/bin/caddy
2021/11/20 11:40:44 [INFO] Cleaning up temporary folder: /tmp/buildenv_2021-11-20-1139.311727266
Restarting the Caddy service
$ service caddy restart
Stopping caddy... done
Starting caddy... done
Log: /var/log/caddy/caddy.log
$ caddy version
v2.4.7-0.20211116200822-7f364c777acf h1:xzMIAbKkKhMYD+zo2AMkRHwzCXxQU3e/fd3foPnv1og=
Checking the process log, the log entry below reassures me that Caddy is now aware of the FreeBSD trust store.
{"level":"info","ts":"2021-11-20T15:09:18.360+0800","logger":"pki.ca.local","msg":"root certificate is already trusted by system","path":"storage:pki/authorities/local/root.crt"}
Where to from here?
I have a few follow-up questions, but for the moment, I’m not sure what I should report back on the SmallStep repo? I’m aware that Caddy uses SmallStep libraries for local HTTPS. Based on post #38 in the thread mTLS under FreeBSD , I understand that I need to report something so that changes to the SmallStep libraries will eventually flow to Caddy. I’m not sure I’ve got the language down pat though.
For instance, for the symlink, I’ve referred to an absolute path for the source path. The service file that sets up Caddy (reproduced below) is aware of the environmental variables XDG_DATA_HOME and XDG_CONFIG_HOME.
#!/bin/sh
# PROVIDE: caddy
# REQUIRE: LOGIN DAEMON NETWORKING
# KEYWORD: shutdown
# To enable caddy, add 'caddy_enable="YES"' to /etc/rc.conf or
# /etc/rc.conf.local
# Optional settings:
# caddy_config (string): Full path to caddy config file
# (/usr/local/etc/caddy/Caddyfile)
# caddy_adapter (string): Config adapter type (caddyfile)
# caddy_directory (string): Root for caddy storage (ACME certs, etc.)
# (/var/db/caddy)
# caddy_extra_flags (string): Extra flags passed to caddy start
# caddy_logdir (string): Where caddy logs are stored
# (/var/log/caddy)
# caddy_logfile (string): Location of process log (${caddy_logdir}/caddy.log)
# This is for startup/shutdown/error messages.
# To create an access log, see:
# https://caddyserver.com/docs/caddyfile/directives/log
# caddy_user (user): User to run caddy (root)
# caddy_group (group): Group to run caddy (wheel)
#
# This script will honor XDG_CONFIG_HOME/XDG_DATA_HOME. Caddy will create a
# .../caddy subdir in each of those. By default, they are subdirs of /var/db/caddy.
# See https://caddyserver.com/docs/conventions#data-directory
. /etc/rc.subr
name=caddy
rcvar=caddy_enable
desc="Powerful, enterprise-ready, open source web server with automatic HTTPS written in Go"
load_rc_config $name
# Defaults
: ${caddy_enable:=NO}
: ${caddy_adapter:=caddyfile}
: ${caddy_config:=/usr/local/etc/caddy/Caddyfile}
: ${caddy_directory:=/var/db/caddy}
: ${caddy_extra_flags:=""}
: ${caddy_logdir:="/var/log/${name}"}
: ${caddy_logfile:="${caddy_logdir}/${name}.log"}
: ${caddy_user:="root"}
: ${caddy_group:="wheel"}
# Config and base directories
: ${XDG_CONFIG_HOME:="${caddy_directory}/config"}
: ${XDG_DATA_HOME:="${caddy_directory}/data"}
export XDG_CONFIG_HOME XDG_DATA_HOME
command="/usr/local/bin/${name}"
caddy_flags="--config ${caddy_config} --adapter ${caddy_adapter}"
pidfile="/var/run/${name}/${name}.pid"
required_files="${caddy_config} ${command}"
start_precmd="caddy_precmd"
start_cmd="caddy_start"
stop_cmd="caddy_stop"
# Extra Commands
extra_commands="configtest reload"
configtest_cmd="caddy_command validate ${caddy_flags}"
reload_cmd="caddy_command reload ${caddy_flags}"
caddy_command()
{
/usr/bin/su -m "${caddy_user}" -c "${command} $*"
}
caddy_precmd()
{
# Create required directories and set permissions
/usr/bin/install -d -m 755 -o "${caddy_user}" -g "${caddy_group}" ${caddy_directory}
/usr/bin/install -d -m 700 -o "${caddy_user}" -g "${caddy_group}" ${caddy_directory}/config
/usr/bin/install -d -m 700 -o "${caddy_user}" -g "${caddy_group}" ${caddy_directory}/data
/usr/bin/install -d -m 755 -o "${caddy_user}" -g "${caddy_group}" ${caddy_logdir}
/usr/bin/install -d -m 700 -o "${caddy_user}" -g "${caddy_group}" /var/run/caddy
}
caddy_start()
{
echo -n "Starting caddy... "
/usr/bin/su -m ${caddy_user} -c "${command} start ${caddy_flags} \
${caddy_extra_flags} --pidfile ${pidfile}" >> ${caddy_logfile} 2>&1
if [ $? -eq 0 ] && ps -ax -o pid | grep -q "$(cat ${pidfile})"; then
echo "done"
echo "Log: ${caddy_logfile}"
else
echo "Error: Caddy failed to start"
echo "Check the caddy log: ${caddy_logfile}"
fi
}
caddy_stop()
{
echo -n "Stopping caddy... "
if caddy_command stop; then
echo "done"
else
echo "Error: Unable to stop caddy"
echo "Check the caddy log: ${caddy_logfile}"
fi
}
run_rc_command "$1"
However, the interactive shell that I use isn’t.
$ setenv
LOGNAME=root
SHELL=/bin/csh
HOME=/root
USER=root
BLOCKSIZE=K
MAIL=/var/mail/root
MM_CHARSET=UTF-8
PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin:/root/bin
PWD=/usr/local/etc/rc.d
TERM=xterm-256color
LANG=en_US.UTF-8
LC_ALL=en_US.UTF-8
HOSTTYPE=FreeBSD
VENDOR=amd
OSTYPE=FreeBSD
MACHTYPE=x86_64
SHLVL=1
GROUP=wheel
HOST=caddy
EDITOR=vi
PAGER=less
So the source path I’m using is absolute for me, but might be different for someone else as the service file suggests it is configurable. If the XDA environmental variables were available to me, Step 1, I’m guessing, might have looked like:
$ mkdir -p /usr/local/etc/ssl/certs
$ ln -s $XDG_DATA_HOME/caddy/pki/authorities/local/root.crt /usr/local/etc/ssl/certs/root.crt
I suppose this would be more universal.
