1. The problem I’m having:
Hello,
Since we switched from 2.4.6 to 2.7.x beta on some boxes, I noticed an issue with “tls internal” certs for the IP Addresses endpoints.
Certificates files for these internal tls targets are residing in the “local” certificates folder, which seems good.
/var/lib/caddy/.local/share/caddy
[root@web23 caddy]# find -name "94*"
./certificates/local/94.103.96.188
./certificates/local/94.103.96.188/94.103.96.188.crt
./certificates/local/94.103.96.188/94.103.96.188.key
./certificates/local/94.103.96.188/94.103.96.188.json
[root@web23 caddy]# cd certificates/local
[root@web23 local]# ls -la
total 0
drwx------ 7 caddy caddy 99 Jun 25 10:00 .
drwx------ 5 caddy caddy 97 Oct 28 2021 ..
drwx------ 2 caddy caddy 52 Oct 14 2021 --1
drwx------ 2 caddy caddy 70 Oct 14 2021 127.0.0.1
drwx------ 2 caddy caddy 100 Oct 14 2021 2a00-a500-0-96--188
drwx------ 2 caddy caddy 82 Oct 14 2021 94.103.96.188
drwx------ 2 caddy caddy 70 Jun 25 10:00 localhost
However, when caddy want to renew them (every 12h), it seems it tries to renew them through LE / ZeroSSL, which fails because it does not qualify for public certs.
Also I see some on_demand logs line about these certificates, when for these, I’ve explicitly set “tls internal” (without the on_demand option)
{"level":"info","ts":1687649361.350319,"logger":"tls.cache.maintenance","msg":"certificate expires soon; queuing for renewal","identifiers":["2a00:a500:0:96::188"],"remaining":13829.649681449}
{"level":"info","ts":1687662561.3478105,"logger":"tls.cache.maintenance","msg":"certificate expires soon; queuing for renewal","identifiers":["94.103.96.188"],"remaining":13941.652192865}
{"level":"info","ts":1687648816.401833,"logger":"tls.on_demand","msg":"obtaining new certificate","remote_ip":"2620:96:e000:b0cc:e:2:2:6","remote_port":"34478","server_name":"2a00:a500:0:96::188"}
{"level":"info","ts":1687648816.4040043,"logger":"tls.obtain","msg":"acquiring lock","identifier":"2a00:a500:0:96::188"}
{"level":"info","ts":1687648816.4086325,"logger":"tls.obtain","msg":"lock acquired","identifier":"2a00:a500:0:96::188"}
{"level":"info","ts":1687648816.408801,"logger":"tls.obtain","msg":"obtaining certificate","identifier":"2a00:a500:0:96::188"}
{"level":"error","ts":1687648816.4125872,"logger":"tls.obtain","msg":"will retry","error":"[2a00:a500:0:96::188] Obtain: subject does not qualify for a public certificate: 2a00:a500:0:96::188","attempt":1,"retrying_in":60,"elapsed":0.003934779,"max_durati$
{"level":"info","ts":1687648876.4133291,"logger":"tls.obtain","msg":"obtaining certificate","identifier":"2a00:a500:0:96::188"}
{"level":"error","ts":1687648876.4137444,"logger":"tls.obtain","msg":"will retry","error":"[2a00:a500:0:96::188] Obtain: subject does not qualify for a public certificate: 2a00:a500:0:96::188","attempt":2,"retrying_in":120,"elapsed":60.005093677,"max_dura$
{"level":"info","ts":1687648996.402827,"logger":"tls.obtain","msg":"releasing lock","identifier":"2a00:a500:0:96::188"}
{"level":"error","ts":1687648996.4030983,"logger":"tls.on_demand","msg":"renewing certificate on-demand failed","subjects":["2a00:a500:0:96::188"],"not_after":1687663191,"error":"context canceled"}
{"level":"error","ts":1687648996.404658,"logger":"tls.on_demand","msg":"renewing certificate on-demand failed","subjects":["2a00:a500:0:96::188"],"not_after":1687663191,"error":"no matching certificate to load for : open /var/lib/caddy/.local/share/caddy/$
{"level":"info","ts":1687648998.1338716,"logger":"tls.on_demand","msg":"obtaining new certificate","remote_ip":"2602:80d:1000:b0cc:e:2:5:6","remote_port":"50130","server_name":"2a00:a500:0:96::188"}
{"level":"info","ts":1687648998.1342003,"logger":"tls.obtain","msg":"acquiring lock","identifier":"2a00:a500:0:96::188"}
{"level":"info","ts":1687648998.135236,"logger":"tls.obtain","msg":"lock acquired","identifier":"2a00:a500:0:96::188"}
{"level":"info","ts":1687648998.1353586,"logger":"tls.obtain","msg":"obtaining certificate","identifier":"2a00:a500:0:96::188"}
{"level":"error","ts":1687648998.135599,"logger":"tls.obtain","msg":"will retry","error":"[2a00:a500:0:96::188] Obtain: subject does not qualify for a public certificate: 2a00:a500:0:96::188","attempt":1,"retrying_in":60,"elapsed":0.00034584,"max_duration$
{"level":"info","ts":1687649058.1365676,"logger":"tls.obtain","msg":"obtaining certificate","identifier":"2a00:a500:0:96::188"}
{"level":"error","ts":1687649058.1368542,"logger":"tls.obtain","msg":"will retry","error":"[2a00:a500:0:96::188] Obtain: subject does not qualify for a public certificate: 2a00:a500:0:96::188","attempt":2,"retrying_in":120,"elapsed":60.001601645,"max_dura$
{"level":"info","ts":1687649178.134662,"logger":"tls.obtain","msg":"releasing lock","identifier":"2a00:a500:0:96::188"}
{"level":"error","ts":1687649178.134821,"logger":"tls.on_demand","msg":"renewing certificate on-demand failed","subjects":["2a00:a500:0:96::188"],"not_after":1687663191,"error":"context canceled"}
{"level":"info","ts":1687662103.291739,"logger":"tls.on_demand","msg":"obtaining new certificate","remote_ip":"94.103.96.130","remote_port":"35378","server_name":"94.103.96.188"}
{"level":"info","ts":1687662103.2955809,"logger":"tls.obtain","msg":"acquiring lock","identifier":"94.103.96.188"}
{"level":"info","ts":1687662103.2965617,"logger":"tls.obtain","msg":"lock acquired","identifier":"94.103.96.188"}
{"level":"info","ts":1687662103.2966921,"logger":"tls.obtain","msg":"obtaining certificate","identifier":"94.103.96.188"}
{"level":"error","ts":1687662103.2970204,"logger":"tls.obtain","msg":"will retry","error":"[94.103.96.188] Obtain: subject does not qualify for a public certificate: 94.103.96.188","attempt":1,"retrying_in":60,"elapsed":0.000441277,"max_duration":2592000}
{"level":"info","ts":1687662163.2976422,"logger":"tls.obtain","msg":"obtaining certificate","identifier":"94.103.96.188"}
{"level":"error","ts":1687662163.2980185,"logger":"tls.obtain","msg":"will retry","error":"[94.103.96.188] Obtain: subject does not qualify for a public certificate: 94.103.96.188","attempt":2,"retrying_in":120,"elapsed":60.001438909,"max_duration":259200$
{"level":"error","ts":1687662262.1381907,"logger":"tls.on_demand","msg":"renewing certificate on-demand failed","subjects":["94.103.96.188"],"not_after":1687676503,"error":"timed out waiting to obtain certificate for 94.103.96.188"}
{"level":"error","ts":1687662283.037333,"logger":"tls.on_demand","msg":"renewing certificate on-demand failed","subjects":["94.103.96.188"],"not_after":1687676503,"error":"timed out waiting to obtain certificate for 94.103.96.188"}
{"level":"info","ts":1687662283.2978654,"logger":"tls.obtain","msg":"releasing lock","identifier":"94.103.96.188"}
{"level":"error","ts":1687662283.2980218,"logger":"tls.on_demand","msg":"renewing certificate on-demand failed","subjects":["94.103.96.188"],"not_after":1687676503,"error":"context canceled"}
{"level":"error","ts":1687662283.2981737,"logger":"tls.on_demand","msg":"renewing certificate on-demand failed","subjects":["94.103.96.188"],"not_after":1687676503,"error":"no matching certificate to load for : open /var/lib/caddy/.local/share/caddy/certi$
{"level":"error","ts":1687662283.2981794,"logger":"tls.on_demand","msg":"renewing certificate on-demand failed","subjects":["94.103.96.188"],"not_after":1687676503,"error":"no matching certificate to load for : open /var/lib/caddy/.local/share/caddy/certi$
{"level":"error","ts":1687662283.2982175,"logger":"tls.on_demand","msg":"renewing certificate on-demand failed","subjects":["94.103.96.188"],"not_after":1687676503,"error":"no matching certificate to load for : open /var/lib/caddy/.local/share/caddy/certi$
{"level":"error","ts":1687662283.2982714,"logger":"tls.on_demand","msg":"renewing certificate on-demand failed","subjects":["94.103.96.188"],"not_after":1687676503,"error":"no matching certificate to load for : open /var/lib/caddy/.local/share/caddy/certi$
{"level":"error","ts":1687662283.2981806,"logger":"tls.on_demand","msg":"renewing certificate on-demand failed","subjects":["94.103.96.188"],"not_after":1687676503,"error":"no matching certificate to load for : open /var/lib/caddy/.local/share/caddy/certi$
{"level":"error","ts":1687662283.2984183,"logger":"tls.on_demand","msg":"renewing certificate on-demand failed","subjects":["94.103.96.188"],"not_after":1687676503,"error":"no matching certificate to load for : open /var/lib/caddy/.local/share/caddy/certi$
{"level":"info","ts":1687662322.1662922,"logger":"tls.on_demand","msg":"obtaining new certificate","remote_ip":"212.243.40.116","remote_port":"58667","server_name":"94.103.96.188"}
{"level":"info","ts":1687662322.166539,"logger":"tls.obtain","msg":"acquiring lock","identifier":"94.103.96.188"}
{"level":"info","ts":1687662322.167656,"logger":"tls.obtain","msg":"lock acquired","identifier":"94.103.96.188"}
{"level":"info","ts":1687662322.1677473,"logger":"tls.obtain","msg":"obtaining certificate","identifier":"94.103.96.188"}
{"level":"error","ts":1687662322.16795,"logger":"tls.obtain","msg":"will retry","error":"[94.103.96.188] Obtain: subject does not qualify for a public certificate: 94.103.96.188","attempt":1,"retrying_in":60,"elapsed":0.000280393,"max_duration":2592000}
{"level":"info","ts":1687662382.169044,"logger":"tls.obtain","msg":"obtaining certificate","identifier":"94.103.96.188"}
{"level":"error","ts":1687662382.1694114,"logger":"tls.obtain","msg":"will retry","error":"[94.103.96.188] Obtain: subject does not qualify for a public certificate: 94.103.96.188","attempt":2,"retrying_in":120,"elapsed":60.001741248,"max_duration":259200$
{"level":"error","ts":1687662463.0567381,"logger":"tls.on_demand","msg":"renewing certificate on-demand failed","subjects":["94.103.96.188"],"not_after":1687676503,"error":"timed out waiting to obtain certificate for 94.103.96.188"}
{"level":"error","ts":1687662492.8766448,"logger":"tls.on_demand","msg":"renewing certificate on-demand failed","subjects":["94.103.96.188"],"not_after":1687676503,"error":"timed out waiting to obtain certificate for 94.103.96.188"}
{"level":"error","ts":1687662502.1429038,"logger":"tls.on_demand","msg":"renewing certificate on-demand failed","subjects":["94.103.96.188"],"not_after":1687676503,"error":"timed out waiting to obtain certificate for 94.103.96.188"}
{"level":"info","ts":1687662502.1664727,"logger":"tls.obtain","msg":"releasing lock","identifier":"94.103.96.188"}
{"level":"error","ts":1687662502.1667027,"logger":"tls.on_demand","msg":"renewing certificate on-demand failed","subjects":["94.103.96.188"],"not_after":1687676503,"error":"context canceled"}
{"level":"error","ts":1687662502.1668742,"logger":"tls.on_demand","msg":"renewing certificate on-demand failed","subjects":["94.103.96.188"],"not_after":1687676503,"error":"no matching certificate to load for : open /var/lib/caddy/.local/share/caddy/certi$
{"level":"error","ts":1687662502.166896,"logger":"tls.on_demand","msg":"renewing certificate on-demand failed","subjects":["94.103.96.188"],"not_after":1687676503,"error":"no matching certificate to load for : open /var/lib/caddy/.local/share/caddy/certif$
{"level":"error","ts":1687662502.1669002,"logger":"tls.on_demand","msg":"renewing certificate on-demand failed","subjects":["94.103.96.188"],"not_after":1687676503,"error":"no matching certificate to load for : open /var/lib/caddy/.local/share/caddy/certi$
{"level":"error","ts":1687662502.1669648,"logger":"tls.on_demand","msg":"renewing certificate on-demand failed","subjects":["94.103.96.188"],"not_after":1687676503,"error":"no matching certificate to load for : open /var/lib/caddy/.local/share/caddy/certi$
{"level":"error","ts":1687662502.1670926,"logger":"tls.on_demand","msg":"renewing certificate on-demand failed","subjects":["94.103.96.188"],"not_after":1687676503,"error":"no matching certificate to load for : open /var/lib/caddy/.local/share/caddy/certi$
{"level":"error","ts":1687662502.1672964,"logger":"tls.on_demand","msg":"renewing certificate on-demand failed","subjects":["94.103.96.188"],"not_after":1687676503,"error":"no matching certificate to load for : open /var/lib/caddy/.local/share/caddy/certi$
{"level":"info","ts":1687662523.0864077,"logger":"tls.on_demand","msg":"obtaining new certificate","remote_ip":"94.103.96.130","remote_port":"38802","server_name":"94.103.96.188"}
{"level":"info","ts":1687662523.086662,"logger":"tls.obtain","msg":"acquiring lock","identifier":"94.103.96.188"}
{"level":"info","ts":1687662523.0877855,"logger":"tls.obtain","msg":"lock acquired","identifier":"94.103.96.188"}
{"level":"info","ts":1687662523.0879154,"logger":"tls.obtain","msg":"obtaining certificate","identifier":"94.103.96.188"}
{"level":"error","ts":1687662523.0881758,"logger":"tls.obtain","msg":"will retry","error":"[94.103.96.188] Obtain: subject does not qualify for a public certificate: 94.103.96.188","attempt":1,"retrying_in":60,"elapsed":0.000364581,"max_duration":2592000}
{"level":"info","ts":1687662561.3478105,"logger":"tls.cache.maintenance","msg":"certificate expires soon; queuing for renewal","identifiers":["94.103.96.188"],"remaining":13941.652192865}
{"level":"info","ts":1687662561.347979,"logger":"tls.cache.maintenance","msg":"attempting certificate renewal","identifiers":["94.103.96.188"],"remaining":13941.652024121}
{"level":"info","ts":1687662561.3482196,"logger":"tls.renew","msg":"acquiring lock","identifier":"94.103.96.188"}
But after a few attemps it finally succeed to renew it (through local cert I guess) and the situation is back to normal.
{"level":"info","ts":1687649361.3794806,"logger":"tls.renew","msg":"certificate renewed successfully","identifier":"2a00:a500:0:96::188"}
{"level":"info","ts":1687662883.5889027,"logger":"tls.renew","msg":"certificate renewed successfully","identifier":"94.103.96.188"}`
The problem is that in the meantime, for a few minutes, the SSL is broken for these endpoints. I noticed this because we monitor the endpoints from Zabbix, and every 12h we have alerts about SSL error on these.
Any idea what could be wrong here ?
When we were running 2.6.4 with same configs, on these hosts, the issue was not present.
Kind regards
2. Full log output
Can’t post it. It exceed the allowed amount of chars that can be posted.
3. Caddy version:
v2.7.0-beta.2 h1:jaS1odoRuDR2W8igaKgVGvVjhTNt8xfoz3YPC4bcenA=
d. My complete Caddy config:
{
admin 127.0.0.1:8888
default_bind 127.0.0.1 [::1] 94.103.96.188 [2a00:a500:0:96::188]
grace_period 3s
log {
output file /var/log/caddy/caddy.log {
roll_size 250MiB
roll_keep_for 15d
}
level ERROR
}
email no@notreally.com
on_demand_tls {
ask https://you.dont.want.to.know/caddy/dnslookup
interval 2m
burst 10000
}
servers {
trusted_proxies cloudflare {
interval 12h
timeout 15s
}
}
}
# Common options we want to apply to every "virtualhosts"
(common) {
@sc_server_fqdn {
path /_sc_get_server_fqdn
}
respond @sc_server_fqdn "web23.swisscenter.com" 200 {
close
}
reverse_proxy http://127.0.0.80:80
}
# Host related endpoints
http://web23.swisscenter.com, http://localhost, http://127.0.0.1, http://[::1], http://94.103.96.188, http://[2a00:a500:0:96::188] {
redir https://web23.swisscenter.com{uri} 308
}
https://localhost, https://127.0.0.1, https://[::1], https://94.103.96.188, https://[2a00:a500:0:96::188] {
tls internal
redir https://web23.swisscenter.com{uri} 308
}
https://web23.swisscenter.com {
import common
}
# LVE Manager endpoint
manager.web23.swisscenter.com {
@manager_access {
not remote_ip 192.168.50.0/24
}
route @manager_access {
respond "We're sorry, but this resource is not available for you. If you feed this is an error, please contact your amazing server administrator." 403 {
close
}
}
reverse_proxy http://127.0.0.1:9000
}
# Per virtualhost specific configs
# NOTE: This folder is currently EMPTY. It's only there if we need to add specific tweaks/config for some specific customer
import /etc/caddy/customers/*.conf
# Default catchall endpoints
http:// {
import common
}
https:// {
import common
tls {
on_demand
load /etc/caddy/certs
}
}
That might be useful.
